Improve memory

.gitignore

@@ -22,6 +22,8 @@ video/auth.*
 *.jpeg
 *.mp4
 
+**/*.log
+
 r5-java/tmp
 property-data
 property-data2

Dockerfile

@@ -39,6 +39,10 @@ VOLUME ["/app/data"]
 
 RUN chown -R appuser:appuser /app
 USER appuser
+# Fallback for any allocations not served by jemalloc (the binary's global
+# allocator, tuned via the baked-in malloc_conf): cap glibc to 2 arenas so freed
+# memory coalesces and is returned instead of fragmenting across per-CPU arenas.
+ENV MALLOC_ARENA_MAX=2
 EXPOSE 8001
 HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
   CMD curl -f http://localhost:8001/health || exit 1

Makefile.data

@@ -64,8 +64,6 @@ PBF             := $(DATA_DIR)/england-latest.osm.pbf
 FR_TOW          := $(DATA_DIR)/FR_TOW_V1_ALL.zip
 NFI             := $(DATA_DIR)/NFI_WOODLAND_ENGLAND.zip
 TREE_DENSITY_PC := $(DATA_DIR)/tree_density_by_postcode.parquet
-TREE_DENSITY_STREETS := $(DATA_DIR)/tree_density_by_street.parquet
-TREE_DENSITY_ADDR := $(DATA_DIR)/tree_density_by_address.parquet
 OFS_REGISTER    := $(DATA_DIR)/ofs_register.xlsx
 PLACES          := $(DATA_DIR)/places.parquet
 MEDIAN_AGE      := $(DATA_DIR)/median_age.parquet
@@ -117,10 +115,10 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
         transform-school-proximity transform-tree-density \
         generate-postcode-boundaries generate-travel-times enrich-actual-listings
 
-prepare:                      $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
-	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)"
+prepare:                      $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles  property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
+	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
 merge:                        $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
-	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ)
+	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
 enrich-actual-listings:       $(ACTUAL_LISTINGS_ENRICHED)
 tiles:                        $(TILES) $(SATELLITE_TILES) $(SATELLITE_HIGHRES_TILES)
 satellite-tiles:              $(SATELLITE_TILES)
@@ -183,6 +181,7 @@ $(PC_BOUNDARIES_STAMP): $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP) $(ARCGI
 		--oa-boundaries $(OA_BOUNDARIES) \
 		--inspire $(INSPIRE_DIR) \
 		--output $(PC_BOUNDARIES)
+	$(VALIDATE_OUTPUTS) --active-postcode-boundary-match "$(ARCGIS)::$(PC_BOUNDARIES)"
 	@touch $@
 generate-travel-times: $(ARCGIS) $(PLACES) $(PBF) download-transit-network
 	@if [ -f "$(R5_NETWORK_CACHE)" ] && { [ "$(PBF)" -nt "$(R5_NETWORK_CACHE)" ] || [ "$(TRANSIT_STAMP)" -nt "$(R5_NETWORK_CACHE)" ]; }; then \
@@ -328,8 +327,8 @@ $(GREENSPACE): $(PBF)
 $(OS_GREENSPACE):
 	uv run python -m pipeline.download.os_greenspace --output $@
 
-$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN) $(OFS_REGISTER) $(ARCGIS)
-	uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN) --university-register $(OFS_REGISTER) --postcodes $(ARCGIS)
+$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN) $(OFS_REGISTER) $(ARCGIS) $(POIS_RAW)
+	uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN) --university-register $(OFS_REGISTER) --postcodes $(ARCGIS) --pois $(POIS_RAW) --include-streets
 
 
 $(MEDIAN_AGE):
@@ -358,7 +357,7 @@ $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(GIAS) $(OFSTE
 $(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
 	uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
 
-$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(PC_BOUNDARIES) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
+$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
 	$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
 	uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR)
 
@@ -368,15 +367,12 @@ $(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DE
 $(SCHOOL_PROX): $(OFSTED) $(ARCGIS) pipeline/transform/school_proximity.py pipeline/utils/poi_counts.py
 	uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
 
-$(TREE_DENSITY_PC): $(FR_TOW) $(NFI) $(ARCGIS) $(PRICE_PAID) $(TREE_DENSITY_DEPS)
+$(TREE_DENSITY_PC): $(FR_TOW) $(NFI) $(ARCGIS) $(TREE_DENSITY_DEPS)
 	uv run python -m pipeline.transform.tree_density \
 		--tow-zip $(FR_TOW) \
 		--nfi-zip $(NFI) \
 		--arcgis $(ARCGIS) \
-		--price-paid $(PRICE_PAID) \
-		--output-postcodes $(TREE_DENSITY_PC) \
-		--output-streets $(TREE_DENSITY_STREETS) \
-		--output-addresses $(TREE_DENSITY_ADDR)
+		--output-postcodes $(TREE_DENSITY_PC)
 
 # Postcode boundaries require manual generation — fail with instructions
 $(PC_BOUNDARIES):
@@ -417,13 +413,13 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
 		--tree-density-postcodes $(TREE_DENSITY_PC) \
 		--output-postcodes $(POSTCODES_PQ) \
 		--output-properties $(PROPERTIES_PQ)
-	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ)
+	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
 	@touch $@
 
 # ── Price estimation (post-merge) ───────────────────────────────────────────
 
 $(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP)
-	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ)
+	$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
 
 $(PRICE_INDEX): $(MERGE_STAMP) $(PRICE_INDEX_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
 	uv run python -m pipeline.transform.price_estimation.index --input $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --output $@
@@ -432,7 +428,7 @@ $(PRICE_INDEX): $(MERGE_STAMP) $(PRICE_INDEX_DEPS) | $(PROPERTIES_PQ) $(POSTCODE
 $(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
 	@rm -f $@
 	uv run python -m pipeline.transform.price_estimation.estimate --properties $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --index $(PRICE_INDEX)
-	$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX)
+	$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
 	@touch $@
 
 $(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \

docker-compose.yml

@@ -32,6 +32,9 @@ services:
             - ./property-data:/app/data:ro
             - ./finder/data:/app/finder-data:ro
         environment:
+            # Fallback only — the binary uses jemalloc as its global allocator
+            # (tuned via a baked-in malloc_conf). Caps glibc to 2 arenas.
+            MALLOC_ARENA_MAX: "2"
             POCKETBASE_URL: http://pocketbase:8090
             POCKETBASE_ADMIN_EMAIL: *pb-email
             POCKETBASE_ADMIN_PASSWORD: *pb-password

finder/Dockerfile

@@ -0,0 +1,25 @@
+# Finder scraper image. Runs via docker-compose sharing the media_gluetun VPN
+# network namespace; the source tree is bind-mounted at runtime, so this image
+# only needs the Python deps. The venv lives OUTSIDE the bind-mount target
+# (/opt/venv) so the mount doesn't shadow it.
+FROM python:3.12-slim
+
+ENV UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    PYTHONUNBUFFERED=1
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends ca-certificates curl \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app/finder
+
+# Install dependencies into /opt/venv (cached layer; project code is mounted at runtime).
+COPY pyproject.toml uv.lock ./
+RUN uv sync --no-install-project --frozen
+
+# Source is bind-mounted over /app/finder by compose. `uv run` uses /opt/venv.
+CMD ["sleep", "infinity"]

finder/constants.py

@@ -6,7 +6,9 @@ REPO_DIR = FINDER_DIR.parent
 
 DATA_DIR = Path(os.environ.get("DATA_DIR", str(FINDER_DIR / "data")))
 ARCGIS_PATH = Path(
-    os.environ.get("ARCGIS_PATH", str(REPO_DIR / "property-data" / "arcgis_data.parquet"))
+    os.environ.get(
+        "ARCGIS_PATH", str(REPO_DIR / "property-data" / "arcgis_data.parquet")
+    )
 )
 PAGE_SIZE = 24
 DELAY_BETWEEN_PAGES = 0.3
@@ -19,6 +21,19 @@ MAX_BEDROOMS = 20  # sanity cap — values above this are almost certainly parsi
 TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
 SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
 RIGHTMOVE_BASE = "https://www.rightmove.co.uk"
+# Detail page (plain HTTPS GET, no Cloudflare). Its window.__PAGE_MODEL embeds
+# propertyData.address.{outcode,incode}, which together form the property's TRUE
+# full postcode — the search API only exposes the outcode. {id} is the numeric
+# listing id from the search response.
+RIGHTMOVE_DETAIL_URL = "https://www.rightmove.co.uk/properties/{id}"
+
+# The Rightmove search API gives only an outcode-level display address, so the
+# true full postcode is recovered from each listing's detail page (see
+# finder/rightmove.py::parse_detail_postcode). One extra GET per listing is a
+# big throughput increase over the ~1000-result-per-outcode search, so detail
+# fetching is gated and capped per outcode (mirrors ZOOPLA_* below). Default ON.
+RIGHTMOVE_FETCH_DETAILS = True  # fetch detail pages for true per-listing postcodes
+RIGHTMOVE_MAX_DETAILS_PER_OUTCODE = 4000  # max detail-page fetches per outcode
 
 # OnTheMarket
 ONTHEMARKET_BASE = "https://www.onthemarket.com"
@@ -26,6 +41,41 @@ ONTHEMARKET_BASE = "https://www.onthemarket.com"
 # Zoopla
 ZOOPLA_BASE = "https://www.zoopla.co.uk"
 
+# Zoopla search cards only carry an outcode-level address, so the full postcode
+# and precise coordinates are scraped from each listing's detail page. These
+# bound that extra work (see finder/zoopla.py and finder/scraper.py).
+ZOOPLA_FETCH_DETAILS = True  # fetch detail pages for precise per-listing postcodes
+ZOOPLA_MAX_DETAILS_PER_OUTCODE = 4000  # max detail-page fetches per outcode
+ZOOPLA_DETAIL_GOTO_TIMEOUT_MS = 1500000  # per detail-page navigation timeout
+# Fraction of a single outcode's wall-clock budget (ZOOPLA_OUTCODE_TIMEOUT_SECONDS)
+# spent fetching details; the remainder is reserved for search pagination so
+# detail fetches can never trip the timeout and discard collected listings.
+ZOOPLA_DETAIL_BUDGET_FRACTION = 0.6
+
+# Gluetun VPN. Network endpoints are env-overridable because they are
+# deployment-specific: when finder runs in a SEPARATE container they use the
+# `gluetun` hostname (defaults below); when finder SHARES gluetun's network
+# namespace (docker-compose.yml, network_mode container:media_gluetun) they
+# become localhost and GLUETUN_PROXY is empty (the shared netns already tunnels
+# all traffic, so no HTTP proxy is needed).
+#   GLUETUN_PROXY="" (empty) => direct connection (no proxy); used in shared-netns.
+GLUETUN_PROXY = os.environ.get("GLUETUN_PROXY", "http://gluetun:8888") or None
+GLUETUN_CONTROL_URL = os.environ.get("GLUETUN_CONTROL_URL", "http://gluetun:8000")
+GLUETUN_API_KEY = "My8AbvnKhfyFdRhpTVfoTfa5DkAMmg8K"
+# Egress-IP rotations to try per Cloudflare challenge. Keep at 0 for Zoopla:
+# rotating among Gluetun's datacenter IPs doesn't clear Cloudflare and would
+# rotate away from the IP a cleared Cloudflare session was bound to, voiding it.
+# Raise only with residential IPs where rotation helps.
+GLUETUN_MAX_ROTATIONS = 0  # max egress-IP rotations per Cloudflare challenge
+
+# Zoopla fetcher: "flaresolverr" (default) solves Cloudflare via the FlareSolverr
+# sidecar (docker-compose.yml) and needs no display/VNC — verified to return the
+# RSC flight stream with postcode + coordinates; "camoufox" drives a local
+# anti-fingerprint browser (needs an interactive solve on datacenter IPs).
+ZOOPLA_FETCHER = os.environ.get("ZOOPLA_FETCHER", "flaresolverr")
+FLARESOLVERR_URL = os.environ.get("FLARESOLVERR_URL", "http://gluetun:8191/v1")
+FLARESOLVERR_MAX_TIMEOUT_MS = 120000  # per-request solve budget; first solve is slow
+
 # Greater London-ish postcode areas. This intentionally uses broad area
 # prefixes so a manual scrape can include central/inner London plus common
 # outer-London and near-London outcodes without maintaining a long borough list.

finder/docker-compose.yml

@@ -0,0 +1,57 @@
+# Finder scraper + FlareSolverr, both sharing the EXISTING media_gluetun VPN
+# container's network namespace. Everything egresses through the VPN, and
+# FlareSolverr solves Zoopla's Cloudflare automatically (no VNC needed).
+#
+# Prerequisites:
+#   - The `media_gluetun` container (qmcgaw/gluetun) is running on this host.
+#     It is managed by a different compose; it is referenced here as external
+#     via network_mode "container:media_gluetun".
+#   - Because these services share gluetun's netns, they reach each other and
+#     gluetun on localhost (flaresolverr :8191, gluetun control :8000) and need
+#     NO published ports (which is exactly why this avoids the dev-container
+#     port-forwarding pain).
+#
+# Usage:
+#   cd finder
+#   docker compose up -d --build flaresolverr finder     # start the sidecars
+#   docker compose exec finder uv run python main.py --source zoopla --outcodes SW9 --test
+#   docker compose exec finder uv run python main.py --source all       # full run
+#   docker compose down
+#
+# NOTE: a manually-started `finder_flaresolverr` container from testing must be
+# removed first (`docker rm -f finder_flaresolverr`) to avoid a name clash.
+
+services:
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: finder_flaresolverr
+    network_mode: "container:media_gluetun"
+    environment:
+      LOG_LEVEL: info
+      TZ: Europe/London
+    restart: unless-stopped
+
+  finder:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: finder-scraper:latest
+    container_name: finder_scraper
+    network_mode: "container:media_gluetun"
+    depends_on:
+      - flaresolverr
+    volumes:
+      - .:/app/finder                          # live-mounted finder source
+      - ../property-data:/app/property-data:ro  # ARCGIS postcode data
+    working_dir: /app/finder
+    environment:
+      # Shared netns: sidecars are on localhost, and the netns already tunnels
+      # all traffic through the VPN, so no HTTP proxy is used.
+      ZOOPLA_FETCHER: flaresolverr
+      FLARESOLVERR_URL: http://localhost:8191/v1
+      GLUETUN_CONTROL_URL: http://localhost:8000
+      GLUETUN_PROXY: ""            # empty => direct (shared netns already tunnels)
+      DATA_DIR: /app/finder/data
+      ARCGIS_PATH: /app/property-data/arcgis_data.parquet
+    restart: "no"
+    command: ["sleep", "infinity"]   # stays up; run scrapes via `docker compose exec`

finder/flaresolverr.py

@@ -0,0 +1,91 @@
+"""FlareSolverr client — fetch Cloudflare-protected pages as rendered HTML.
+
+FlareSolverr (https://github.com/FlareSolverr/FlareSolverr) drives an
+undetected browser to pass Cloudflare's challenge and returns the fully
+rendered HTML. It runs as a sidecar service (see docker-compose.yml) sharing
+the Gluetun VPN network namespace, so its browser egresses through the VPN.
+
+Verified working against Zoopla's managed Turnstile on a datacenter VPN IP,
+provided a reused session and a generous maxTimeout (~120s) — the first
+challenge solve is slow, subsequent requests on the warm session are fast.
+"""
+
+import logging
+
+import httpx
+
+from constants import FLARESOLVERR_MAX_TIMEOUT_MS, FLARESOLVERR_URL
+
+log = logging.getLogger("flaresolverr")
+
+
+class FlareSolverrError(Exception):
+    """Raised when FlareSolverr cannot fetch/solve a URL."""
+
+
+class FlareSolverrSession:
+    """A reusable FlareSolverr browser session (context manager).
+
+    Reusing one session keeps the cleared Cloudflare cookies warm across
+    requests, so only the first fetch pays the full challenge-solve cost."""
+
+    def __init__(
+        self,
+        url: str = FLARESOLVERR_URL,
+        session: str = "finder",
+        max_timeout_ms: int = FLARESOLVERR_MAX_TIMEOUT_MS,
+    ) -> None:
+        self._url = url
+        self._session = session
+        self._max_timeout = max_timeout_ms
+        # Read timeout must comfortably exceed maxTimeout (FlareSolverr blocks
+        # for up to maxTimeout while solving before responding).
+        self._client = httpx.Client(timeout=httpx.Timeout(self._max_timeout / 1000 + 30))
+        self._active = False
+
+    def _post(self, payload: dict) -> dict:
+        try:
+            resp = self._client.post(self._url, json=payload)
+            resp.raise_for_status()
+            data = resp.json()
+        except (httpx.HTTPError, ValueError) as exc:
+            raise FlareSolverrError(
+                f"FlareSolverr request to {self._url} failed: {exc}"
+            ) from exc
+        if data.get("status") != "ok":
+            raise FlareSolverrError(
+                f"FlareSolverr {payload.get('cmd')} failed: {data.get('message')}"
+            )
+        return data
+
+    def __enter__(self) -> "FlareSolverrSession":
+        # Start from a clean session (ignore destroy errors for a fresh name).
+        try:
+            self._post({"cmd": "sessions.destroy", "session": self._session})
+        except FlareSolverrError:
+            pass
+        self._post({"cmd": "sessions.create", "session": self._session})
+        self._active = True
+        log.info("FlareSolverr session %r ready at %s", self._session, self._url)
+        return self
+
+    def get(self, url: str) -> str:
+        """Fetch a URL through FlareSolverr; return the solved HTML."""
+        data = self._post(
+            {
+                "cmd": "request.get",
+                "session": self._session,
+                "url": url,
+                "maxTimeout": self._max_timeout,
+            }
+        )
+        solution = data.get("solution") or {}
+        return solution.get("response", "") or ""
+
+    def __exit__(self, *exc_info) -> None:
+        if self._active:
+            try:
+                self._post({"cmd": "sessions.destroy", "session": self._session})
+            except FlareSolverrError as exc:
+                log.debug("FlareSolverr session destroy failed: %s", exc)
+        self._client.close()

finder/gdal-ecw/Dockerfile

@@ -0,0 +1,53 @@
+# GDAL with ECW (read) support, for decoding Environment Agency Vertical Aerial
+# Photography in the satellite-highres pipeline (pipeline/download/satellite_highres.py).
+#
+# EA VAP ships as ECW **v2** rasters, which are readable by the open-source
+# libecwj2 3.3 SDK -- the same library the official OSGeo image uses when built
+# with WITH_ECW=yes. We therefore avoid the proprietary, login-gated Hexagon
+# ERDAS ECW/JP2 SDK (which is only needed for ECW v3) and its licensing
+# restrictions entirely.
+#
+# We build only the ECW driver as a GDAL *plugin* on top of the official runtime
+# image (no full GDAL rebuild). The plugin's GDAL sources are pinned to the exact
+# commit reported by the base image so libgdal and the plugin stay ABI-compatible.
+#
+# Build:  docker build -t perfect-postcode/gdal-ecw:latest docker/gdal-ecw
+# Verify: docker run --rm perfect-postcode/gdal-ecw:latest gdalinfo --formats | grep -i ECW
+
+FROM ghcr.io/osgeo/gdal:ubuntu-full-latest
+
+ARG LIBECWJ2_URL=https://github.com/rouault/libecwj2-3.3-builds/releases/download/v1/install-libecwj2-3.3-ubuntu-20.04.tar.gz
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cmake g++ make git curl ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Open-source ECW v2 SDK (extracts to /opt/libecwj2-3.3) + make its libs loadable.
+RUN curl --retry 3 --retry-all-errors --retry-delay 3 -fsSL -o /tmp/libecwj2.tar.gz "$LIBECWJ2_URL" \
+    && tar -C / -xzf /tmp/libecwj2.tar.gz \
+    && rm -f /tmp/libecwj2.tar.gz \
+    && (cd /opt/libecwj2-3.3/lib && for so in *.so*; do \
+            ln -sf "/opt/libecwj2-3.3/lib/$so" "/usr/lib/x86_64-linux-gnu/$so"; \
+        done) \
+    && ldconfig
+
+# Build the ECW driver plugin against the base image's exact GDAL sources.
+RUN set -eux; \
+    GDAL_COMMIT="$(gdalinfo --version | sed -nE 's/.*-([0-9a-f]{8,}).*/\1/p')"; \
+    test -n "$GDAL_COMMIT"; \
+    echo "Building ECW plugin for GDAL commit ${GDAL_COMMIT}"; \
+    mkdir -p /tmp/gdal && cd /tmp/gdal && git init -q; \
+    git fetch --depth 1 -q https://github.com/OSGeo/gdal.git "$GDAL_COMMIT"; \
+    git checkout -q FETCH_HEAD; \
+    cmake -S frmts/ecw -B /tmp/ecw-build \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_PREFIX_PATH=/usr \
+        -DECW_ROOT=/opt/libecwj2-3.3; \
+    cmake --build /tmp/ecw-build -j"$(nproc)"; \
+    PLUGIN_DIR=/usr/lib/x86_64-linux-gnu/gdalplugins; \
+    mkdir -p "$PLUGIN_DIR"; \
+    find /tmp/ecw-build -name 'gdal_ECW*.so' -exec cp {} "$PLUGIN_DIR/" \; ; \
+    rm -rf /tmp/gdal /tmp/ecw-build
+
+# Fail the build if the driver is not actually available.
+RUN gdalinfo --formats | grep -iq 'ECW.*rw' && echo "ECW driver OK"

finder/http_client.py

@@ -5,7 +5,7 @@ import time
 import httpx
 from fake_useragent import UserAgent
 
-from constants import MAX_RETRIES, RETRY_BASE_DELAY
+from constants import GLUETUN_PROXY, MAX_RETRIES, RETRY_BASE_DELAY
 
 log = logging.getLogger("rightmove")
 
@@ -15,10 +15,12 @@ _ua = UserAgent(
 
 
 def make_client() -> httpx.Client:
+    # Route through the Gluetun HTTP proxy (VPN egress) when configured.
     return httpx.Client(
         timeout=30,
         headers={"User-Agent": _ua.random, "Accept": "application/json"},
         follow_redirects=True,
+        proxy=GLUETUN_PROXY or None,
     )
 
 

finder/main.py

@@ -57,6 +57,16 @@ def parse_args() -> argparse.Namespace:
         default=DATA_DIR,
         help=f"Directory for parquet output. Defaults to {DATA_DIR}.",
     )
+    parser.add_argument(
+        "--outcodes",
+        type=str,
+        default=None,
+        help=(
+            "Comma-separated outcodes to scrape (e.g. 'SW9' or 'SW9,E14,BR1') "
+            "instead of the full Greater London set. Must fall within the "
+            "London-ish areas; takes precedence over --test/--limit-outcodes."
+        ),
+    )
     parser.add_argument(
         "--limit-outcodes",
         type=int,
@@ -116,10 +126,25 @@ def main() -> int:
     from scraper import (
         build_postcode_coords,
         build_postcode_index,
+        filter_londonish_outcodes,
         load_outcodes,
         run_scrape,
     )
 
+    if args.outcodes is not None:
+        requested = [code.strip().upper() for code in args.outcodes.split(",") if code.strip()]
+        if not requested:
+            raise SystemExit("--outcodes was empty")
+        outcodes = filter_londonish_outcodes(requested)
+        dropped = sorted(set(requested) - set(outcodes))
+        if dropped:
+            log.warning("Ignoring outcodes outside the Greater London-ish areas: %s", ", ".join(dropped))
+        if not outcodes:
+            raise SystemExit(
+                "None of the requested outcodes are within the Greater London-ish areas "
+                f"({', '.join(requested)})."
+            )
+    else:
         outcodes = load_outcodes()
         if args.test and args.limit_outcodes is None:
             preferred = [outcode for outcode in TEST_OUTCODES if outcode in set(outcodes)]

finder/onthemarket.py

@@ -10,6 +10,30 @@ Each rendered page contains 30 listings under
 `humanised-property-type`, `features` (a list where the first element is
 typically `"Tenure: <value>"`), and `details-url`. Pagination is via
 `?page=N`; the loop terminates when `paginationControls.next` is null.
+
+Postcodes
+---------
+The search card exposes only an *outcode*-level address (e.g. "Padfield Road,
+London, SE5") and a map pin, so the old behaviour derived the postcode from the
+nearest postcode to that pin — a guess that frequently lands on a neighbouring
+unit (the pin can sit on the wrong side of a street boundary).
+
+Each *detail* page (`/details/{id}/`) is a plain HTTPS GET whose `__NEXT_DATA__`
+embeds the property's analytics dataLayer at
+`props.initialReduxState.metadata.dataLayer`, which carries the property's own
+`postcode` (full unit postcode, e.g. "SE5 9AA") keyed to this listing by
+`property-id`. Crucially this is NOT the agent's office postcode — that lives
+separately at `…property.agent.postcode` ("SE5 8RS" for the same listing) and
+is the classic trap when blindly scanning the page for a postcode. We read the
+dataLayer postcode, verify `property-id` matches the listing, and accept it only
+when its outcode agrees with the coordinate-nearest postcode (via
+``resolve_listing_postcode``) — exactly the trust rule the other scrapers use.
+Measured over a sample of real listings this yields a trustworthy, usually
+exact-unit postcode for ~11/12 listings; the rest safely fall back to the
+coordinate-nearest postcode.
+
+Detail fetching costs one extra HTTPS GET per listing, so it is gated behind
+``OTM_FETCH_DETAILS`` and capped at ``OTM_MAX_DETAILS_PER_OUTCODE`` per outcode.
 """
 
 import json
@@ -31,14 +55,26 @@ from spatial import PostcodeSpatialIndex
 from transform import (
     clean_listing_address,
     extract_full_postcode,
+    extract_outcode,
     fix_coords,
     map_property_type,
     normalize_sub_type,
     parse_display_size,
+    resolve_listing_postcode,
 )
 
 log = logging.getLogger("rightmove")
 
+# Detail-page postcode recovery (see module docstring). When enabled, each
+# listing's detail page is fetched so its analytics dataLayer postcode — the
+# property's own full unit postcode — can replace the coordinate-nearest guess.
+# Bounded per outcode so a large outcode can't balloon into unbounded extra
+# HTTPS GETs. Kept at parity with the Rightmove/Zoopla detail caps (400) so a
+# typical outcode's listings all get their real postcode rather than a
+# coordinate-nearest guess.
+OTM_FETCH_DETAILS = True
+OTM_MAX_DETAILS_PER_OUTCODE = 400
+
 _NEXT_DATA_RE = re.compile(
     r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
     re.DOTALL,
@@ -51,6 +87,11 @@ _HTML_HEADERS = {
     "Accept-Language": "en-GB,en;q=0.9",
 }
 
+# listingId -> recovered full postcode (or None). Failures are cached too so a
+# broken or postcode-less detail page is not re-fetched within a run (the same
+# listing can reappear across overlapping outcode searches).
+_detail_postcode_cache: dict[str, str | None] = {}
+
 
 def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict | None:
     """GET one search-results page and return the embedded __NEXT_DATA__ JSON.
@@ -119,6 +160,116 @@ def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict
     return None
 
 
+def parse_detail_postcode(html: str, listing_id: str | None = None) -> str | None:
+    """Extract the property's own full postcode from an OnTheMarket detail page.
+
+    Pure and network-free so it is unit-testable: callers pass `page.content()`
+    / the GET body and this does the parsing.
+
+    The postcode lives in the analytics dataLayer embedded in `__NEXT_DATA__` at
+    ``props.initialReduxState.metadata.dataLayer.postcode`` and is the
+    property's own unit postcode (e.g. "SE5 9AA"). It is deliberately NOT the
+    agent's office postcode, which sits separately at
+    ``…property.agent.postcode`` — the trap when scanning a detail page for "a"
+    postcode. When ``listing_id`` is given, the dataLayer's ``property-id`` must
+    match it, guaranteeing we read this listing's postcode and not a stray one.
+
+    Returns a normalized full postcode (e.g. "SE5 9AA") or ``None`` when the
+    page has no usable property postcode. Trust (outcode-vs-coordinates
+    agreement) is enforced later in ``transform_property``.
+    """
+    if not html:
+        return None
+
+    match = _NEXT_DATA_RE.search(html)
+    if not match:
+        return None
+    try:
+        data = json.loads(match.group(1))
+    except json.JSONDecodeError:
+        return None
+
+    try:
+        data_layer = data["props"]["initialReduxState"]["metadata"]["dataLayer"]
+    except (KeyError, TypeError):
+        return None
+    if not isinstance(data_layer, dict):
+        return None
+
+    # Guard against reading a different listing's postcode: the dataLayer is the
+    # property's own analytics payload, so its property-id must match.
+    if listing_id is not None:
+        page_id = data_layer.get("property-id")
+        if page_id is not None and str(page_id) != str(listing_id):
+            return None
+
+    raw_postcode = data_layer.get("postcode")
+    if not isinstance(raw_postcode, str):
+        return None
+    return extract_full_postcode(raw_postcode)
+
+
+def _fetch_detail_postcode(
+    client: httpx.Client, details_url: str, listing_id: str
+) -> str | None:
+    """GET one listing's detail page and return its dataLayer postcode (or None).
+
+    Results (including failures) are cached by listing id so a listing that
+    reappears across overlapping outcode searches is fetched at most once. Plain
+    HTTPS GET — OnTheMarket detail pages have no Cloudflare challenge. Network /
+    parse errors degrade gracefully to None so the caller falls back to the
+    coordinate-nearest postcode.
+    """
+    if listing_id in _detail_postcode_cache:
+        return _detail_postcode_cache[listing_id]
+
+    full_url = (
+        ONTHEMARKET_BASE + details_url
+        if details_url and not details_url.startswith("http")
+        else details_url
+    )
+    result: str | None = None
+    if full_url:
+        for attempt in range(MAX_RETRIES):
+            try:
+                resp = client.get(
+                    full_url, headers=_HTML_HEADERS, follow_redirects=True
+                )
+            except (
+                httpx.ConnectError,
+                httpx.ReadTimeout,
+                httpx.WriteTimeout,
+                httpx.PoolTimeout,
+            ) as exc:
+                delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
+                log.warning(
+                    "%s from %s, retry %d/%d in %.1fs",
+                    type(exc).__name__, full_url, attempt + 1, MAX_RETRIES, delay,
+                )
+                time.sleep(delay)
+                continue
+
+            if resp.status_code == 200:
+                result = parse_detail_postcode(resp.text, listing_id)
+                break
+            if resp.status_code in (429, 500, 502, 503, 504):
+                delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
+                log.warning(
+                    "HTTP %d from %s, retry %d/%d in %.1fs",
+                    resp.status_code, full_url, attempt + 1, MAX_RETRIES, delay,
+                )
+                time.sleep(delay)
+                continue
+            log.debug(
+                "OnTheMarket detail %s returned HTTP %d (no postcode)",
+                listing_id, resp.status_code,
+            )
+            break
+
+    _detail_postcode_cache[listing_id] = result
+    return result
+
+
 def _parse_price(price_value) -> int:
     """Parse a formatted price string like '£450,000' into an integer.
     Returns 0 for POA/auction/null values."""
@@ -166,9 +317,19 @@ def _extract_floor_area(features: list) -> float | None:
 
 
 def transform_property(
-    raw: dict, pc_index: PostcodeSpatialIndex
+    raw: dict,
+    pc_index: PostcodeSpatialIndex,
+    detail_postcode: str | None = None,
 ) -> dict | None:
-    """Transform a raw OnTheMarket listing dict into our output schema."""
+    """Transform a raw OnTheMarket listing dict into our output schema.
+
+    ``detail_postcode`` is the property's own full postcode recovered from its
+    detail page (see ``parse_detail_postcode`` / ``_fetch_detail_postcode``),
+    or ``None`` when no detail fetch was done / no postcode was found. When
+    present and trustworthy (its outcode agrees with the coordinate-nearest
+    postcode) it supersedes the coordinate guess and is labelled
+    ``"detail_address"``.
+    """
     loc = raw.get("location") or {}
     raw_lat = loc.get("lat")
     raw_lng = loc.get("lon")
@@ -184,8 +345,29 @@ def transform_property(
         return None
     raw_address = raw.get("address", "") or ""
     extracted_postcode = extract_full_postcode(raw_address)
-    postcode = extracted_postcode or inferred_postcode
-    postcode_source = "address" if extracted_postcode else "coordinates"
+
+    # Prefer the property's own detail-page postcode when we have one and it is
+    # trustworthy. The detail postcode is a full unit postcode (better than the
+    # coordinate-nearest guess and than the usually outcode-only card address),
+    # but a stale/mislabelled value would silently override the spatially
+    # correct one, so apply the same outcode-agreement trust rule the address
+    # postcode uses: keep it only when its outcode matches the
+    # coordinate-nearest postcode's outcode.
+    detail_postcode = extract_full_postcode(detail_postcode)
+    if detail_postcode and extract_outcode(detail_postcode) == extract_outcode(
+        inferred_postcode
+    ):
+        postcode, postcode_source = detail_postcode, "detail_address"
+    else:
+        if detail_postcode:
+            log.debug(
+                "OnTheMarket %s: rejecting detail postcode %s "
+                "(outcode mismatch with inferred %s)",
+                raw.get("id", "?"), detail_postcode, inferred_postcode,
+            )
+        postcode, postcode_source = resolve_listing_postcode(
+            extracted_postcode, inferred_postcode
+        )
 
     raw_beds = raw.get("bedrooms") or 0
     raw_baths = raw.get("bathrooms") or 0
@@ -223,6 +405,10 @@ def transform_property(
         "Inferred postcode": inferred_postcode,
         "Listing raw address": raw_address,
         "Address per Property Register": clean_listing_address(raw_address),
+        # OnTheMarket search JSON exposes only a street-level address; no UPRN
+        # or house number/name is available without a detail-page fetch.
+        "UPRN": None,
+        "Property number or name": None,
         "Leasehold/Freehold": _extract_tenure(features),
         "Property type": map_property_type(sub_type),
         "Property sub-type": normalize_sub_type(sub_type),
@@ -242,10 +428,17 @@ def search_outcode(
     pc_index: PostcodeSpatialIndex,
     max_properties: int | None = None,
 ) -> list[dict]:
-    """Paginate through OnTheMarket sale results for one outcode."""
+    """Paginate through OnTheMarket sale results for one outcode.
+
+    When ``OTM_FETCH_DETAILS`` is enabled, up to
+    ``OTM_MAX_DETAILS_PER_OUTCODE`` listings per outcode have their detail page
+    fetched for the property's own postcode (see ``_fetch_detail_postcode``);
+    the rest fall back to the coordinate-nearest postcode.
+    """
     properties: list[dict] = []
     seen_ids: set[str] = set()
     page_num = 1
+    details_fetched = 0
 
     while True:
         data = _fetch_page_json(client, outcode, page_num)
@@ -269,8 +462,22 @@ def search_outcode(
             if listing_id and listing_id in seen_ids:
                 continue
             seen_ids.add(listing_id)
+
+            detail_postcode = None
+            if OTM_FETCH_DETAILS and listing_id:
+                # Cached lookups are free; only fresh GETs count toward the cap
+                # and incur the inter-request delay.
+                cached = listing_id in _detail_postcode_cache
+                if cached or details_fetched < OTM_MAX_DETAILS_PER_OUTCODE:
+                    detail_postcode = _fetch_detail_postcode(
+                        client, raw.get("details-url") or "", listing_id
+                    )
+                    if not cached:
+                        details_fetched += 1
+                        time.sleep(DELAY_BETWEEN_PAGES)
+
             try:
-                transformed = transform_property(raw, pc_index)
+                transformed = transform_property(raw, pc_index, detail_postcode)
             except Exception as exc:
                 log.warning(
                     "OnTheMarket %s property %s failed to transform: %s",

finder/rightmove.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import re
 import time
 
 import httpx
@@ -6,12 +8,15 @@ import httpx
 from constants import (
     PAGE_SIZE,
     DELAY_BETWEEN_PAGES,
+    RIGHTMOVE_DETAIL_URL,
+    RIGHTMOVE_FETCH_DETAILS,
+    RIGHTMOVE_MAX_DETAILS_PER_OUTCODE,
     SEARCH_URL,
     TYPEAHEAD_URL,
 )
 from http_client import fetch_with_retry
 from spatial import PostcodeSpatialIndex
-from transform import transform_property
+from transform import extract_full_postcode, normalize_postcode, transform_property
 
 log = logging.getLogger("rightmove")
 
@@ -23,6 +28,176 @@ outcode_cache: dict[str, str] = {}
 _MAX_INDEX = 1008
 
 
+# ---------------------------------------------------------------------------
+# Detail-page postcode extraction
+# ---------------------------------------------------------------------------
+#
+# The search API (_paginate) only returns an outcode-level `displayAddress`
+# (e.g. "Akerman Road, Brixton, London, SW9") — never the full postcode. Each
+# listing's detail page, however, embeds the property's OWN full postcode in a
+# `window.__PAGE_MODEL` script as `propertyData.address.{outcode, incode}`
+# (e.g. outcode "SW9" + incode "0HD" → "SW9 0HD"), independently corroborated by
+# `propertyData.propertyUrls.similarPropertiesUrl` ("/property-for-sale/SW9-0HD.html").
+# This is the property's own postcode, NOT a nearest station/school: the
+# `nearestStations`/`nearestAirports` arrays carry only names + distances, no
+# postcodes, and the address outcode always matches the searched outcode.
+# Recon over 24 live listings across SW9/E1/M1/LS6/E20 (incl. APPROXIMATE_POINT
+# new-builds) found the full postcode present 100% of the time. There is no
+# UPRN or house-number field anywhere in propertyData, so those stay None.
+#
+# __PAGE_MODEL is a "devalue"-style flattened object graph: its `data` field is
+# a JSON STRING holding a flat array where every integer inside a container is
+# an index reference into that same array (so the graph can dedupe). We
+# brace-match the (large, deeply-nested) object literal — a non-greedy regex
+# cannot — then rehydrate the reference graph before reading the address.
+
+_PAGE_MODEL_RE = re.compile(r"window\.__PAGE_MODEL\s*=\s*")
+
+
+def _extract_page_model_literal(html: str) -> str | None:
+    """Return the `{...}` object literal assigned to window.__PAGE_MODEL.
+
+    Brace-matches with string/escape awareness so embedded braces and quotes in
+    string values don't end the match early. Returns None when absent."""
+    marker = _PAGE_MODEL_RE.search(html)
+    if not marker:
+        return None
+    start = marker.end()
+    if start >= len(html) or html[start] != "{":
+        return None
+    depth = 0
+    in_str = False
+    esc = False
+    for j in range(start, len(html)):
+        ch = html[j]
+        if in_str:
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                in_str = False
+        elif ch == '"':
+            in_str = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return html[start : j + 1]
+    return None
+
+
+def _rehydrate(flat: list) -> object:
+    """Resolve a devalue-style flattened reference array into a nested object.
+
+    Index 0 is the root; every int inside a dict/list is an index back into
+    ``flat``. Memoised so shared/cyclic references resolve once."""
+    cache: dict[int, object] = {}
+
+    def resolve(idx: int) -> object:
+        if not isinstance(idx, int) or idx < 0 or idx >= len(flat):
+            return None
+        if idx in cache:
+            return cache[idx]
+        node = flat[idx]
+        if isinstance(node, dict):
+            out: dict = {}
+            cache[idx] = out
+            for key, value in node.items():
+                out[key] = resolve(value) if isinstance(value, int) else value
+            return out
+        if isinstance(node, list):
+            arr: list = []
+            cache[idx] = arr
+            for value in node:
+                arr.append(resolve(value) if isinstance(value, int) else value)
+            return arr
+        cache[idx] = node
+        return node
+
+    return resolve(0)
+
+
+def parse_detail_postcode(html: str) -> str | None:
+    """Extract a Rightmove property's TRUE full postcode from its detail HTML.
+
+    Pure and network-free so it is unit-testable: callers pass the page HTML.
+    Reads ``propertyData.address.outcode`` + ``.incode`` from window.__PAGE_MODEL
+    and returns a normalised full postcode (e.g. "SW9 0HD"), or None when the
+    page has no parseable address (the property location wrapper can be empty —
+    the caller then keeps the coordinate fallback). The returned outcode is
+    re-validated against the joined postcode so a malformed incode is dropped.
+    """
+    if not html:
+        return None
+    literal = _extract_page_model_literal(html)
+    if not literal:
+        return None
+    try:
+        outer = json.loads(literal)
+        flat = json.loads(outer["data"])
+    except (ValueError, KeyError, TypeError):
+        return None
+    if not isinstance(flat, list) or not flat:
+        return None
+
+    root = _rehydrate(flat)
+    if not isinstance(root, dict):
+        return None
+    property_data = root.get("propertyData")
+    if not isinstance(property_data, dict):
+        return None
+    address = property_data.get("address")
+    if not isinstance(address, dict):
+        return None
+
+    outcode = address.get("outcode")
+    incode = address.get("incode")
+    if not isinstance(outcode, str) or not isinstance(incode, str):
+        return None
+    outcode, incode = outcode.strip(), incode.strip()
+    if not outcode or not incode:
+        return None
+
+    # Round-trip through the shared postcode validator/normaliser: this both
+    # canonicalises spacing and rejects an outcode/incode pair that doesn't form
+    # a structurally-valid UK postcode.
+    return extract_full_postcode(normalize_postcode(f"{outcode} {incode}"))
+
+
+# listingId -> true full postcode (or None when unavailable). Failures are
+# cached too, so a broken/duplicate listing is fetched at most once per run (the
+# same listing can reappear across overlapping outcode searches).
+_detail_postcode_cache: dict[str, str | None] = {}
+
+
+def _fetch_detail_postcode(client: httpx.Client, property_id: str) -> str | None:
+    """GET a listing detail page and return its true full postcode (or None).
+
+    Results (including failures) are cached by listing id. The detail page is a
+    plain HTML GET — no Cloudflare, unlike Zoopla — so a single httpx call
+    suffices; any error degrades gracefully to the coordinate fallback."""
+    if not property_id:
+        return None
+    if property_id in _detail_postcode_cache:
+        return _detail_postcode_cache[property_id]
+
+    postcode: str | None = None
+    url = RIGHTMOVE_DETAIL_URL.format(id=property_id)
+    try:
+        resp = client.get(url, headers={"Accept": "text/html"})
+        if resp.status_code == 200:
+            postcode = parse_detail_postcode(resp.text)
+        else:
+            log.debug("Rightmove detail %s returned HTTP %d", url, resp.status_code)
+    except httpx.HTTPError as exc:
+        log.debug("Rightmove detail fetch failed %s: %s", url, exc)
+
+    _detail_postcode_cache[property_id] = postcode
+    return postcode
+
+
 def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
     """Look up Rightmove's internal ID for an outcode via typeahead API."""
     if outcode in outcode_cache:
@@ -44,6 +219,31 @@ def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
     return None
 
 
+def _detail_postcode_for(
+    client: httpx.Client,
+    prop: dict,
+    fetch_details: bool,
+    detail_budget: dict,
+) -> str | None:
+    """Look up a listing's true postcode, honouring the per-outcode fetch cap.
+
+    Cached listings are always served (they cost neither a cap slot nor a GET);
+    a fresh fetch is made only while ``detail_budget['remaining'] > 0``."""
+    if not fetch_details:
+        return None
+    property_id = str(prop.get("id") or "")
+    if not property_id:
+        return None
+    if property_id in _detail_postcode_cache:
+        return _detail_postcode_cache[property_id]
+    if detail_budget["remaining"] <= 0:
+        return None
+    detail_budget["remaining"] -= 1
+    postcode = _fetch_detail_postcode(client, property_id)
+    time.sleep(DELAY_BETWEEN_PAGES)
+    return postcode
+
+
 def _paginate(
     client: httpx.Client,
     outcode_id: str,
@@ -51,11 +251,19 @@ def _paginate(
     channel_cfg: dict,
     pc_index: PostcodeSpatialIndex,
     max_properties: int | None = None,
+    fetch_details: bool = False,
+    detail_cap: int = 0,
 ) -> tuple[list[dict], int]:
-    """Paginate through search results. Returns (properties, result_count)."""
+    """Paginate through search results. Returns (properties, result_count).
+
+    When ``fetch_details`` is set, up to ``detail_cap`` listings per outcode have
+    their detail page fetched for the property's TRUE full postcode (see
+    ``parse_detail_postcode``); the rest fall back to coordinate-derived
+    postcodes."""
     properties = []
     index = 0
     result_count = 0
+    detail_budget = {"remaining": detail_cap}
 
     while True:
         params = {
@@ -82,7 +290,12 @@ def _paginate(
 
         for prop in raw_props:
             try:
-                transformed = transform_property(prop, outcode, pc_index)
+                detail_postcode = _detail_postcode_for(
+                    client, prop, fetch_details, detail_budget
+                )
+                transformed = transform_property(
+                    prop, outcode, pc_index, detail_postcode=detail_postcode
+                )
             except Exception as exc:
                 log.warning(
                     "Rightmove %s/%s property %s failed to transform: %s",
@@ -127,7 +340,12 @@ def search_outcode(
     pc_index: PostcodeSpatialIndex,
     max_properties: int | None = None,
 ) -> list[dict]:
-    """Paginate through unfiltered sale results for one outcode+channel."""
+    """Paginate through unfiltered sale results for one outcode+channel.
+
+    Each listing's detail page is fetched for the property's TRUE full postcode
+    (gated by ``RIGHTMOVE_FETCH_DETAILS`` and capped per outcode by
+    ``RIGHTMOVE_MAX_DETAILS_PER_OUTCODE``); listings beyond the cap keep the
+    coordinate-derived postcode."""
     properties, _ = _paginate(
         client,
         outcode_id,
@@ -135,6 +353,8 @@ def search_outcode(
         channel_cfg,
         pc_index,
         max_properties=max_properties,
+        fetch_details=RIGHTMOVE_FETCH_DETAILS,
+        detail_cap=RIGHTMOVE_MAX_DETAILS_PER_OUTCODE,
     )
 
     if max_properties is not None and len(properties) >= max_properties:

finder/scraper.py

@@ -15,6 +15,10 @@ from constants import (
     DATA_DIR,
     DELAY_BETWEEN_OUTCODES,
     LONDON_OUTCODE_PREFIXES,
+    ZOOPLA_DETAIL_BUDGET_FRACTION,
+    ZOOPLA_FETCH_DETAILS,
+    ZOOPLA_FETCHER,
+    ZOOPLA_MAX_DETAILS_PER_OUTCODE,
 )
 
 from http_client import make_client
@@ -371,6 +375,36 @@ def _zoopla_outcode_timeout_seconds() -> int:
     return timeout
 
 
+def _zoopla_detail_cap() -> int:
+    """Max detail-page fetches per outcode (0 disables detail fetching).
+
+    Zoopla search cards only expose an outcode-level address, so the full
+    postcode/coordinates come from each listing's detail page. The cap bounds
+    the extra page loads so an outcode stays within ZOOPLA_OUTCODE_TIMEOUT_SECONDS
+    (the per-outcode SIGALRM budget covers the detail fetches too). Configure via
+    ZOOPLA_FETCH_DETAILS / ZOOPLA_MAX_DETAILS_PER_OUTCODE in constants.py."""
+    return ZOOPLA_MAX_DETAILS_PER_OUTCODE if ZOOPLA_FETCH_DETAILS else 0
+
+
+def _open_zoopla_detail_tab(page, detail_cap: int):
+    """Open a second tab on the same context for detail-page fetches.
+
+    Sharing the persistent context means the detail tab inherits the search
+    tab's Cloudflare clearance cookies. Returns None when detail fetching is
+    disabled or the tab cannot be created (the scrape then degrades to
+    outcode-level postcodes rather than failing)."""
+    if detail_cap <= 0:
+        return None
+    try:
+        return page.context.new_page()
+    except Exception as exc:
+        log.warning(
+            "Zoopla detail tab unavailable (%s); using outcode-level postcodes",
+            _exception_detail(exc),
+        )
+        return None
+
+
 @contextmanager
 def _wall_clock_timeout(seconds: int, label: str):
     """SIGALRM-based wall-clock guard (POSIX). Raises OutcodeTimeout on expiry.
@@ -438,6 +472,50 @@ def _close_zoopla_browser(browser, label: str) -> None:
         log.warning("%s browser force-close failed: %s", label, _exception_detail(exc))
 
 
+def _scrape_zoopla_flaresolverr(
+    outcodes: list[str],
+    pc_index: PostcodeSpatialIndex,
+    pc_coords: dict[str, tuple[float, float]],
+    results: dict[str, list[dict]],
+    errors: list[str],
+    max_properties_per_source: int | None,
+) -> None:
+    """Scrape Zoopla via the FlareSolverr sidecar (no browser/VNC)."""
+    from flaresolverr import FlareSolverrError, FlareSolverrSession
+    from zoopla_flaresolverr import search_outcode as fs_search_outcode
+
+    try:
+        session = FlareSolverrSession(session="zoopla")
+        session.__enter__()
+    except FlareSolverrError as exc:
+        errors.append(f"zoopla: FlareSolverr unavailable: {exc}")
+        log.warning("Zoopla skipped: FlareSolverr unavailable: %s", exc)
+        return
+
+    try:
+        for outcode in outcodes:
+            remaining = _source_remaining(results, "zoopla", max_properties_per_source)
+            if remaining == 0:
+                log.info("Zoopla cap reached")
+                return
+            try:
+                props, _ = fs_search_outcode(
+                    outcode,
+                    pc_index,
+                    pc_coords,
+                    session,
+                    max_properties=remaining,
+                    detail_cap=ZOOPLA_MAX_DETAILS_PER_OUTCODE,
+                )
+                added = _store_properties(results, "zoopla", props, max_properties_per_source)
+                log.info("Zoopla %s: +%d", outcode, added)
+            except Exception as exc:  # noqa: BLE001 - one outcode must not kill the run
+                _record_error(errors, "zoopla", outcode, exc)
+            time.sleep(DELAY_BETWEEN_OUTCODES)
+    finally:
+        session.__exit__(None, None, None)
+
+
 def _scrape_zoopla(
     outcodes: list[str],
     pc_index: PostcodeSpatialIndex,
@@ -446,6 +524,12 @@ def _scrape_zoopla(
     errors: list[str],
     max_properties_per_source: int | None,
 ) -> None:
+    if ZOOPLA_FETCHER == "flaresolverr":
+        _scrape_zoopla_flaresolverr(
+            outcodes, pc_index, pc_coords, results, errors, max_properties_per_source
+        )
+        return
+
     try:
         browser, page = _launch_zoopla_with_retries()
     except Exception as exc:
@@ -454,6 +538,12 @@ def _scrape_zoopla(
         return
 
     outcode_timeout = _zoopla_outcode_timeout_seconds()
+    detail_cap = _zoopla_detail_cap()
+    detail_page = _open_zoopla_detail_tab(page, detail_cap)
+    # Spend at most a fraction of each outcode's budget on detail fetches so the
+    # SIGALRM guard never trips mid-outcode and discards already-collected
+    # search listings; the rest is left for search pagination and transform.
+    detail_budget_seconds = max(10.0, outcode_timeout * ZOOPLA_DETAIL_BUDGET_FRACTION)
 
     try:
         for outcode in outcodes:
@@ -470,6 +560,9 @@ def _scrape_zoopla(
                             pc_index,
                             pc_coords,
                             max_properties=None,
+                            detail_page=detail_page,
+                            detail_cap=detail_cap,
+                            detail_budget_seconds=detail_budget_seconds,
                         )
                     added = _store_properties(
                         results,
@@ -496,6 +589,8 @@ def _scrape_zoopla(
                     _close_zoopla_browser(browser, f"zoopla {outcode}")
                     try:
                         browser, page = _launch_zoopla_with_retries()
+                        # The old context (and its detail tab) is gone; reopen one.
+                        detail_page = _open_zoopla_detail_tab(page, detail_cap)
                         log.info("Zoopla %s retrying with fresh browser", outcode)
                     except Exception as relaunch_exc:
                         _record_error(errors, "zoopla", outcode, relaunch_exc)
@@ -503,6 +598,11 @@ def _scrape_zoopla(
 
             time.sleep(DELAY_BETWEEN_OUTCODES)
     finally:
+        if detail_page is not None:
+            try:
+                detail_page.close()
+            except Exception:
+                pass
         _close_zoopla_browser(browser, "zoopla final")
 
 

finder/storage.py

@@ -126,6 +126,14 @@ def write_parquet(properties: list[dict], path: Path) -> None:
             "Address per Property Register": [
                 p["Address per Property Register"] for p in properties
             ],
+            # UPRN (when the scraper recovered it) keys an exact listing->EPC
+            # join; Property number or name is the house identifier for the
+            # Price-Paid address join. Both are None for sources/listings without
+            # a detail-page fetch.
+            "UPRN": [p.get("UPRN") for p in properties],
+            "Property number or name": [
+                p.get("Property number or name") for p in properties
+            ],
             "Leasehold/Freehold": [p["Leasehold/Freehold"] for p in properties],
             "Property type": [p["Property type"] for p in properties],
             "Property sub-type": [p["Property sub-type"] for p in properties],
@@ -149,6 +157,8 @@ def write_parquet(properties: list[dict], path: Path) -> None:
             "Inferred postcode": pl.Utf8,
             "Listing raw address": pl.Utf8,
             "Address per Property Register": pl.Utf8,
+            "UPRN": pl.Utf8,
+            "Property number or name": pl.Utf8,
             "Leasehold/Freehold": pl.Utf8,
             "Property type": pl.Utf8,
             "Property sub-type": pl.Utf8,

finder/test_onthemarket.py

@@ -0,0 +1,206 @@
+"""Tests for the OnTheMarket scraper's detail-page postcode recovery.
+
+`parse_detail_postcode` is pure (takes the detail-page HTML, returns a postcode
+or None), so these tests use a trimmed but faithful copy of a real OnTheMarket
+detail page's `__NEXT_DATA__` payload. The fixture mirrors the live structure:
+the property's own postcode lives in the analytics dataLayer
+(`props.initialReduxState.metadata.dataLayer.postcode`) while the agent's office
+postcode sits separately under `…property.agent.postcode` — the trap we must not
+fall into.
+"""
+
+import json
+
+import onthemarket
+from onthemarket import parse_detail_postcode, transform_property
+
+
+class _StubIndex:
+    """Minimal stand-in for PostcodeSpatialIndex returning a fixed postcode."""
+
+    def __init__(self, postcode: str | None):
+        self._postcode = postcode
+
+    def nearest(self, lat: float, lng: float) -> str | None:
+        return self._postcode
+
+
+def _detail_html(
+    *,
+    property_id: int = 19522441,
+    datalayer_postcode: str = "SE5 9AA",
+    agent_postcode: str = "SE5 8RS",
+) -> str:
+    """Build detail-page HTML with a real-shaped __NEXT_DATA__ payload."""
+    next_data = {
+        "props": {
+            "initialReduxState": {
+                "metadata": {
+                    "dataLayer": {
+                        "page-type": "details-section",
+                        "property-type": "homes",
+                        # The property's own unit postcode.
+                        "postcode": datalayer_postcode,
+                        "property-id": property_id,
+                        "price": "275,000",
+                        "addressline_2": "Padfield Road",
+                    }
+                },
+                "property": {
+                    "displayAddress": "Padfield Road, London, SE5",
+                    "location": {"lon": -0.100233, "lat": 51.466129},
+                    # The agent block carries the AGENT'S office postcode — the
+                    # trap. parse_detail_postcode must not return this.
+                    "agent": {
+                        "address": "29 Denmark Hill, Camberwell\nLondon\nSE5 8RS",
+                        "postcode": agent_postcode,
+                    },
+                },
+            }
+        }
+    }
+    payload = json.dumps(next_data)
+    return (
+        "<html><body>"
+        '<script id="__NEXT_DATA__" type="application/json">'
+        f"{payload}"
+        "</script></body></html>"
+    )
+
+
+# ---------------------------------------------------------------------------
+# parse_detail_postcode
+# ---------------------------------------------------------------------------
+
+
+def test_parse_returns_property_postcode_not_agent():
+    html = _detail_html(datalayer_postcode="SE5 9AA", agent_postcode="SE5 8RS")
+    assert parse_detail_postcode(html, "19522441") == "SE5 9AA"
+
+
+def test_parse_normalizes_spacing():
+    html = _detail_html(datalayer_postcode="se59aa")
+    assert parse_detail_postcode(html, "19522441") == "SE5 9AA"
+
+
+def test_parse_ignores_mismatched_property_id():
+    # dataLayer postcode belongs to property 19522441; asking for a different
+    # listing id must refuse to return it.
+    html = _detail_html(property_id=19522441)
+    assert parse_detail_postcode(html, "99999999") is None
+
+
+def test_parse_accepts_when_no_listing_id_given():
+    html = _detail_html(datalayer_postcode="SE5 9AA")
+    assert parse_detail_postcode(html, None) == "SE5 9AA"
+
+
+def test_parse_handles_missing_postcode():
+    html = _detail_html(datalayer_postcode="")
+    assert parse_detail_postcode(html, "19522441") is None
+
+
+def test_parse_handles_no_next_data():
+    assert parse_detail_postcode("<html><body>no script here</body></html>", "1") is None
+
+
+def test_parse_handles_empty_html():
+    assert parse_detail_postcode("", "1") is None
+
+
+def test_parse_handles_malformed_json():
+    html = (
+        '<script id="__NEXT_DATA__" type="application/json">{not json}</script>'
+    )
+    assert parse_detail_postcode(html, "1") is None
+
+
+def test_parse_handles_missing_datalayer():
+    next_data = {"props": {"initialReduxState": {"metadata": {}}}}
+    html = (
+        '<script id="__NEXT_DATA__" type="application/json">'
+        f"{json.dumps(next_data)}</script>"
+    )
+    assert parse_detail_postcode(html, "1") is None
+
+
+# ---------------------------------------------------------------------------
+# transform_property — detail postcode wiring + trust rule
+# ---------------------------------------------------------------------------
+
+
+_RAW_LISTING = {
+    "id": "19522441",
+    "address": "Padfield Road, London, SE5",
+    "location": {"lon": -0.100233, "lat": 51.466129},
+    "bedrooms": 2,
+    "bathrooms": 1,
+    "price": "£275,000",
+    "humanised-property-type": "Apartment",
+    "features": ["Tenure: Leasehold (99 years remaining)"],
+    "details-url": "/details/19522441/",
+}
+
+
+def test_transform_uses_trusted_detail_postcode():
+    # Detail postcode SE5 9AA, coordinate-nearest SE5 1AA: same outcode -> trust
+    # the (more precise) detail postcode and label it detail_address.
+    index = _StubIndex("SE5 1AA")
+    out = transform_property(_RAW_LISTING, index, detail_postcode="SE5 9AA")
+    assert out is not None
+    assert out["Postcode"] == "SE5 9AA"
+    assert out["Postcode source"] == "detail_address"
+
+
+def test_transform_rejects_detail_postcode_on_outcode_mismatch():
+    # Detail postcode SW9 6BZ but coordinate-nearest is SE5 1AA: different
+    # outcode -> reject the detail postcode, fall back to coordinate logic.
+    index = _StubIndex("SE5 1AA")
+    out = transform_property(_RAW_LISTING, index, detail_postcode="SW9 6BZ")
+    assert out is not None
+    assert out["Postcode"] == "SE5 1AA"
+    assert out["Postcode source"] == "coordinates"
+
+
+def test_transform_without_detail_postcode_uses_coordinates():
+    index = _StubIndex("SE5 1AA")
+    out = transform_property(_RAW_LISTING, index, detail_postcode=None)
+    assert out is not None
+    assert out["Postcode"] == "SE5 1AA"
+    assert out["Postcode source"] == "coordinates"
+    # No UPRN / house number is recoverable from OnTheMarket.
+    assert out["UPRN"] is None
+    assert out["Property number or name"] is None
+
+
+def test_transform_detail_postcode_via_search_address_outcode():
+    # When the card address already carries a full postcode that agrees with the
+    # coordinates, the existing "address" source still wins absent a detail
+    # postcode — detail recovery never regresses that path.
+    raw = dict(_RAW_LISTING, address="Padfield Road, London, SE5 1AA")
+    index = _StubIndex("SE5 1AA")
+    out = transform_property(raw, index, detail_postcode=None)
+    assert out["Postcode"] == "SE5 1AA"
+    assert out["Postcode source"] == "address"
+
+
+# ---------------------------------------------------------------------------
+# _fetch_detail_postcode caching (no real network)
+# ---------------------------------------------------------------------------
+
+
+def test_fetch_detail_postcode_is_cached(monkeypatch):
+    onthemarket._detail_postcode_cache.clear()
+    onthemarket._detail_postcode_cache["19522441"] = "SE5 9AA"
+
+    def _boom(*args, **kwargs):  # pragma: no cover - must never be called
+        raise AssertionError("network was hit despite a cached value")
+
+    # Any httpx use would explode; the cache hit must short-circuit first.
+    result = onthemarket._fetch_detail_postcode(
+        client=type("C", (), {"get": _boom})(),
+        details_url="/details/19522441/",
+        listing_id="19522441",
+    )
+    assert result == "SE5 9AA"
+    onthemarket._detail_postcode_cache.clear()

finder/test_rightmove.py

@@ -0,0 +1,113 @@
+"""Tests for the Rightmove detail-page postcode extractor.
+
+The search API only returns an outcode-level ``displayAddress``; the property's
+TRUE full postcode lives on its detail page inside ``window.__PAGE_MODEL`` as
+``propertyData.address.{outcode, incode}``. ``parse_detail_postcode`` recovers
+it. These tests build a faithful __PAGE_MODEL: a devalue-style flattened object
+graph whose ``data`` field is a JSON STRING of a flat array where every integer
+inside a container is an index reference into that same array.
+"""
+
+import json
+
+from rightmove import _extract_page_model_literal, parse_detail_postcode
+
+
+def _page_model_html(flat: list, *, encoding: str = "json") -> str:
+    """Wrap a flattened object-graph array in a realistic detail-page <script>.
+
+    Mirrors the live page: ``window.__PAGE_MODEL = {"data": "<json array>"}``
+    where the array is itself JSON-encoded (so its quotes arrive escaped)."""
+    outer = {"data": json.dumps(flat, separators=(",", ":")), "encoding": encoding}
+    return (
+        "<html><head></head><body>\n"
+        "<script>\n"
+        "    window.__PAGE_MODEL = " + json.dumps(outer, separators=(",", ":")) + ";\n"
+        "</script>\n"
+        "</body></html>"
+    )
+
+
+# A faithful slice of a real listing: root -> propertyData -> address, with a
+# decoy nearestStations array (which carries NO postcodes on the live page) to
+# prove the parser anchors on the property's own address, not a nearby POI.
+_FLAT_SW9 = [
+    {"propertyData": 1},  # 0: root
+    {
+        "id": "89089584",
+        "address": 2,
+        "location": 4,
+        "nearestStations": 6,
+    },  # 1: propertyData
+    {
+        "displayAddress": "Caldwell Street, Stockwell",
+        "countryCode": "GB",
+        "ukCountry": "England",
+        "outcode": "SW9",
+        "incode": "0HD",
+    },  # 2: address
+    None,  # 3: filler
+    {
+        "latitude": 51.477238,
+        "longitude": -0.116819,
+        "pinType": "ACCURATE_POINT",
+    },  # 4: location
+    None,  # 5: filler
+    [7, 8],  # 6: nearestStations (references)
+    {"name": "Oval Station", "distance": 0.36},  # 7: station, no postcode
+    {"name": "Stockwell Station", "distance": 0.41},  # 8: station, no postcode
+]
+
+
+def test_parses_full_postcode_from_outcode_and_incode() -> None:
+    html = _page_model_html(_FLAT_SW9)
+    assert parse_detail_postcode(html) == "SW9 0HD"
+
+
+def test_extract_page_model_literal_brace_matches_nested_object() -> None:
+    # The literal must include the whole nested object, not stop at the first
+    # closing brace inside the escaped data string.
+    html = _page_model_html(_FLAT_SW9)
+    literal = _extract_page_model_literal(html)
+    assert literal is not None
+    assert literal.startswith("{") and literal.endswith("}")
+    # Round-trips back to a dict with the expected top-level keys.
+    assert set(json.loads(literal)) == {"data", "encoding"}
+
+
+def test_normalises_unspaced_incode() -> None:
+    flat = [dict(node) if isinstance(node, dict) else node for node in _FLAT_SW9]
+    flat[2] = {**_FLAT_SW9[2], "outcode": "e20", "incode": "1fh"}
+    assert parse_detail_postcode(_page_model_html(flat)) == "E20 1FH"
+
+
+def test_returns_none_when_address_missing() -> None:
+    # The location wrapper can be empty/absent on some listings; the caller then
+    # keeps the coordinate fallback, so we must return None (not raise).
+    flat = [
+        {"propertyData": 1},
+        {"id": "1", "location": 2},
+        {"latitude": 51.5, "longitude": -0.1},
+    ]
+    assert parse_detail_postcode(_page_model_html(flat)) is None
+
+
+def test_returns_none_when_incode_blank() -> None:
+    flat = [dict(node) if isinstance(node, dict) else node for node in _FLAT_SW9]
+    flat[2] = {**_FLAT_SW9[2], "incode": ""}
+    assert parse_detail_postcode(_page_model_html(flat)) is None
+
+
+def test_returns_none_for_non_postcode_pair() -> None:
+    # A structurally-invalid outcode/incode pair is rejected by the validator.
+    flat = [dict(node) if isinstance(node, dict) else node for node in _FLAT_SW9]
+    flat[2] = {**_FLAT_SW9[2], "outcode": "NOTAPC", "incode": "ZZ"}
+    assert parse_detail_postcode(_page_model_html(flat)) is None
+
+
+def test_returns_none_without_page_model() -> None:
+    assert parse_detail_postcode("") is None
+    assert parse_detail_postcode("<html><body>no model</body></html>") is None
+    # Malformed JSON in the data field degrades gracefully.
+    broken = '<script>window.__PAGE_MODEL = {"data":"[not json"};</script>'
+    assert parse_detail_postcode(broken) is None

finder/test_transform.py

@@ -1,13 +1,19 @@
 from transform import (
+    build_register_address,
     clean_listing_address,
     extract_full_postcode,
+    extract_outcode,
+    resolve_listing_postcode,
     transform_property,
 )
 
 
 class StubPostcodeIndex:
+    def __init__(self, postcode: str = "SW1A 9ZZ") -> None:
+        self._postcode = postcode
+
     def nearest(self, lat: float, lng: float) -> str:
-        return "SW1A 9ZZ"
+        return self._postcode
 
 
 def test_extract_full_postcode_normalizes_spacing() -> None:
@@ -24,6 +30,46 @@ def test_clean_listing_address_removes_postcode_and_outcode_suffixes() -> None:
     assert clean_listing_address("Kings Avenue, Bromley") == "Kings Avenue, Bromley"
 
 
+def test_build_register_address_prepends_house_number_or_name() -> None:
+    # House number/name prepended, with the trailing outcode/postcode stripped.
+    assert (
+        build_register_address("South Street, Bromley BR1", "12")
+        == "12, South Street, Bromley"
+    )
+    assert (
+        build_register_address("Riverside, Martham NR29", "Martham Mill")
+        == "Martham Mill, Riverside, Martham"
+    )
+    # No number/name -> identical to the plain cleaned address.
+    assert build_register_address("Kings Avenue, Bromley", None) == "Kings Avenue, Bromley"
+    # Already starts with the number/name -> no duplication.
+    assert (
+        build_register_address("12 South Street, Bromley", "12")
+        == "12 South Street, Bromley"
+    )
+    # Empty/whitespace number/name is ignored.
+    assert build_register_address("Kings Avenue, Bromley", "  ") == "Kings Avenue, Bromley"
+
+
+def test_extract_outcode() -> None:
+    assert extract_outcode("SW1A 2AA") == "SW1A"
+    assert extract_outcode("n4 2ha") == "N4"
+    assert extract_outcode("SW1A2AA") == "SW1A"
+    assert extract_outcode(None) is None
+    assert extract_outcode("") is None
+
+
+def test_resolve_listing_postcode() -> None:
+    # Outcode matches -> trust the more precise extracted postcode.
+    assert resolve_listing_postcode("SW1A 2AA", "SW1A 9ZZ") == ("SW1A 2AA", "address")
+    # Outcode mismatch -> fall back to the spatially-correct inferred postcode.
+    assert resolve_listing_postcode("E14 9SS", "SW1A 9ZZ") == ("SW1A 9ZZ", "coordinates")
+    # Well-formed but fabricated postcode in a different outcode is rejected.
+    assert resolve_listing_postcode("ZZ9 9ZZ", "SW1A 9ZZ") == ("SW1A 9ZZ", "coordinates")
+    # No extracted postcode -> inferred is authoritative.
+    assert resolve_listing_postcode(None, "SW1A 9ZZ") == ("SW1A 9ZZ", "coordinates")
+
+
 def test_rightmove_transform_prefers_postcode_from_display_address() -> None:
     prop = {
         "id": "123",
@@ -46,3 +92,84 @@ def test_rightmove_transform_prefers_postcode_from_display_address() -> None:
     assert result["Inferred postcode"] == "SW1A 9ZZ"
     assert result["Listing raw address"] == "Flat 2, 10 Downing Street, SW1A 2AA"
     assert result["Address per Property Register"] == "Flat 2, 10 Downing Street"
+
+
+def test_rightmove_transform_rejects_postcode_from_wrong_outcode() -> None:
+    prop = {
+        "id": "124",
+        "location": {"latitude": 51.5, "longitude": -0.1},
+        "price": {"amount": 750000, "displayPrices": []},
+        "propertySubType": "Terraced",
+        "bedrooms": 3,
+        "bathrooms": 1,
+        "keyFeatures": [],
+        "propertyUrl": "/properties/124",
+        # Address postcode is in a different outcode than the coordinate-nearest one.
+        "displayAddress": "10 Downing Street, E14 9SS",
+    }
+
+    result = transform_property(prop, "SW1A", StubPostcodeIndex())
+
+    assert result is not None
+    # The spatially-correct inferred postcode wins over the mismatching extracted one.
+    assert result["Postcode"] == "SW1A 9ZZ"
+    assert result["Postcode source"] == "coordinates"
+    assert result["Extracted postcode"] == "E14 9SS"
+
+
+def _rightmove_prop() -> dict:
+    return {
+        "id": "200",
+        "location": {"latitude": 51.5, "longitude": -0.1},
+        "price": {"amount": 750000, "displayPrices": []},
+        "propertySubType": "Terraced",
+        "bedrooms": 3,
+        "bathrooms": 1,
+        "keyFeatures": [],
+        "propertyUrl": "/properties/200",
+        # Search API only ever exposes the outcode in the display address.
+        "displayAddress": "Caldwell Street, Stockwell, SW9",
+    }
+
+
+def test_rightmove_transform_prefers_detail_postcode() -> None:
+    # The detail page's true full postcode (same outcode as the location) is
+    # preferred over the coordinate-nearest guess.
+    result = transform_property(
+        _rightmove_prop(),
+        "SW9",
+        StubPostcodeIndex("SW9 7AA"),
+        detail_postcode="SW9 0HD",
+    )
+
+    assert result is not None
+    assert result["Postcode"] == "SW9 0HD"
+    assert result["Postcode source"] == "detail_address"
+    # The coordinate inference is still surfaced separately.
+    assert result["Inferred postcode"] == "SW9 7AA"
+
+
+def test_rightmove_transform_rejects_detail_postcode_from_wrong_outcode() -> None:
+    # A detail postcode whose outcode disagrees with the location must not
+    # relocate the listing; the coordinate postcode wins instead.
+    result = transform_property(
+        _rightmove_prop(),
+        "SW9",
+        StubPostcodeIndex("SW9 7AA"),
+        detail_postcode="E14 9SS",
+    )
+
+    assert result is not None
+    assert result["Postcode"] == "SW9 7AA"
+    assert result["Postcode source"] == "coordinates"
+
+
+def test_rightmove_transform_without_detail_keeps_coordinate_logic() -> None:
+    # No detail postcode -> behaviour is unchanged (coordinate-nearest).
+    result = transform_property(
+        _rightmove_prop(), "SW9", StubPostcodeIndex("SW9 7AA")
+    )
+
+    assert result is not None
+    assert result["Postcode"] == "SW9 7AA"
+    assert result["Postcode source"] == "coordinates"

finder/test_zoopla.py

@@ -0,0 +1,288 @@
+from zoopla import _detail_cache_key, parse_detail_geo, transform_property
+
+
+def test_detail_cache_key_uses_listing_id() -> None:
+    assert _detail_cache_key("/for-sale/details/59888978/") == "59888978"
+    assert _detail_cache_key("https://www.zoopla.co.uk/for-sale/details/59888978/") == "59888978"
+    # No id in the URL -> fall back to the URL itself as the key.
+    assert _detail_cache_key("/for-sale/property/br1/") == "/for-sale/property/br1/"
+
+
+class StubPostcodeIndex:
+    """Spatial index stub whose nearest-lookup returns a fixed postcode."""
+
+    def __init__(self, postcode: str = "BR1 2AB") -> None:
+        self._postcode = postcode
+
+    def nearest(self, lat: float, lng: float) -> str:
+        return self._postcode
+
+
+# London-ish postcodes with coordinates, plus the Norfolk sample used by the
+# verified detail-page snippet (well inside the England bounds check).
+PC_COORDS = {
+    "BR1 2AB": (51.40, 0.01),
+    "SW1A 1AA": (51.50, -0.14),
+    "NR29 4RG": (52.716014, 1.614495),
+}
+
+# Verified RSC `location` object (listing 59888978), as it appears escaped inside
+# a self.__next_f flight chunk in page.content().
+_LOCATION_ESCAPED = (
+    '<script>self.__next_f.push([1,"...'
+    '\\"location\\":{\\"outcode\\":\\"NR29\\",'
+    '\\"coordinates\\":{\\"latitude\\":52.716014,\\"longitude\\":1.614495},'
+    '\\"uprn\\":\\"10023461458\\",\\"postalCode\\":\\"NR29 4RG\\",'
+    '\\"propertyNumberOrName\\":\\"Martham Mill\\"}'
+    '..."])</script>'
+)
+
+
+def test_parse_detail_geo_location_object_escaped() -> None:
+    geo = parse_detail_geo(_LOCATION_ESCAPED, search_outcode="NR29")
+    assert geo == {
+        "lat": 52.716014,
+        "lng": 1.614495,
+        "postcode": "NR29 4RG",
+        "outcode": "NR29",
+        "source": "detail_location",
+        "uprn": "10023461458",
+        "number_or_name": "Martham Mill",
+        # No `address` twin in this snippet, so there is no full street address.
+        "full_address": None,
+    }
+
+
+def test_parse_detail_geo_location_object_unescaped() -> None:
+    html = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"uprn":"10023461458","postalCode":"NR29 4RG"}'
+    )
+    geo = parse_detail_geo(html)
+    assert geo is not None
+    assert geo["source"] == "detail_location"
+    assert geo["postcode"] == "NR29 4RG"
+
+
+def test_parse_detail_geo_address_twin() -> None:
+    html = (
+        '"address":{"fullAddress":"Riverside, Martham NR29",'
+        '"latitude":52.716014,"longitude":1.614495,'
+        '"outcode":"NR29","postcode":"NR29 4RG","uprn":"10023461458"}'
+    )
+    geo = parse_detail_geo(html)
+    assert geo is not None
+    assert geo["source"] == "detail_address_obj"
+    assert (geo["lat"], geo["lng"], geo["postcode"]) == (52.716014, 1.614495, "NR29 4RG")
+    assert geo["uprn"] == "10023461458"
+    assert geo["full_address"] == "Riverside, Martham NR29"
+
+
+def test_parse_detail_geo_merges_location_uprn_with_address_full_address() -> None:
+    # Real detail pages carry both wrappers: the `location` object holds the
+    # uprn + house number/name, the `address` twin holds the full street
+    # address. They share a uprn, so the twin's fullAddress is attached.
+    html = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"uprn":"10023461458","postalCode":"NR29 4RG",'
+        '"propertyNumberOrName":"Martham Mill"}'
+        '"address":{"fullAddress":"Riverside, Martham NR29",'
+        '"latitude":52.716014,"longitude":1.614495,'
+        '"outcode":"NR29","postcode":"NR29 4RG","uprn":"10023461458"}'
+    )
+    geo = parse_detail_geo(html)
+    assert geo is not None
+    assert geo["source"] == "detail_location"
+    assert geo["uprn"] == "10023461458"
+    assert geo["number_or_name"] == "Martham Mill"
+    assert geo["full_address"] == "Riverside, Martham NR29"
+
+
+def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None:
+    # The only `address` twin on the page belongs to a different uprn (a
+    # comparable listing). With a uprn to match on, an unrelated twin is never
+    # borrowed — full_address stays None rather than grabbing the wrong street.
+    html = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"uprn":"10023461458","postalCode":"NR29 4RG"}'
+        '"address":{"fullAddress":"Some Comparable, Elsewhere EN2",'
+        '"latitude":51.65,"longitude":-0.08,"uprn":"99999999"}'
+    )
+    geo = parse_detail_geo(html)
+    assert geo is not None
+    assert geo["uprn"] == "10023461458"
+    assert geo["full_address"] is None
+
+
+def test_parse_detail_geo_ignores_poi_coordinates() -> None:
+    # A charger POI (its coordinates NOT wrapped in a "location" object) followed
+    # by the property's own "location" wrapper. Anchoring on the wrapper means
+    # the POI's coordinates are ignored and the property's are returned.
+    poi = (
+        '"name":"Martham Community Centre","numberOfConnectors":2,'
+        '"postcode":"NR29 4SN","coordinates":{"latitude":52.699379,"longitude":1.62921}'
+    )
+    prop = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"uprn":"10023461458","postalCode":"NR29 4RG"}'
+    )
+    geo = parse_detail_geo(poi + prop)
+    assert geo is not None
+    assert geo["source"] == "detail_location"
+    # The property's coords win, not the community centre's.
+    assert (geo["lat"], geo["lng"]) == (52.716014, 1.614495)
+    assert geo["postcode"] == "NR29 4RG"
+
+
+def test_parse_detail_geo_prefers_location_matching_search_outcode() -> None:
+    # Page embeds two location objects (e.g. a comparable then the property).
+    # With a search outcode, the one in that outcode is preferred; without one,
+    # the first (document order = primary listing) is returned.
+    comparable = (
+        '"location":{"outcode":"EN2",'
+        '"coordinates":{"latitude":51.65,"longitude":-0.08},'
+        '"postalCode":"EN2 6SN"}'
+    )
+    target = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"postalCode":"NR29 4RG"}'
+    )
+    geo = parse_detail_geo(comparable + target, search_outcode="NR29")
+    assert geo is not None and geo["postcode"] == "NR29 4RG"
+    geo_first = parse_detail_geo(comparable + target)
+    assert geo_first is not None and geo_first["postcode"] == "EN2 6SN"
+
+
+def test_parse_detail_geo_rejects_out_of_england() -> None:
+    html = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":10.0,"longitude":10.0},'
+        '"uprn":"1","postalCode":"NR29 4RG"}'
+    )
+    assert parse_detail_geo(html) is None
+
+
+def test_parse_detail_geo_drops_inconsistent_postcode() -> None:
+    # postalCode outcode (AB12) disagrees with the object's own outcode (NR29):
+    # keep the coordinates, drop the untrustworthy postcode.
+    html = (
+        '"location":{"outcode":"NR29",'
+        '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
+        '"uprn":"1","postalCode":"AB12 3CD"}'
+    )
+    geo = parse_detail_geo(html)
+    assert geo is not None
+    assert geo["lat"] == 52.716014
+    assert geo["postcode"] is None
+
+
+def test_parse_detail_geo_returns_none_for_garbage() -> None:
+    assert parse_detail_geo("<html><body>no data here</body></html>") is None
+    assert parse_detail_geo("") is None
+    # Coordinates that are not inside a property location/address wrapper (e.g.
+    # only an unwrapped POI) yield nothing — safe degradation to the outcode.
+    assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None
+
+
+def _raw(**overrides) -> dict:
+    raw = {
+        "id": "123",
+        "url": "/for-sale/details/123/",
+        "address": "South Street, Bromley BR1",
+        "price": 500000,
+        "beds": 2,
+        "baths": 1,
+        "property_type": "Flat",
+    }
+    raw.update(overrides)
+    return raw
+
+
+def test_transform_uses_detail_coordinates_with_agreeing_postcode() -> None:
+    detail = {"lat": 51.401, "lng": 0.011, "postcode": "BR1 3CD", "outcode": "BR1"}
+    result = transform_property(
+        _raw(), StubPostcodeIndex("BR1 2AB"), PC_COORDS, search_outcode="BR1", detail=detail
+    )
+    assert result is not None
+    # Extracted detail postcode agrees with the coordinate-nearest outcode -> trusted.
+    assert result["Postcode"] == "BR1 3CD"
+    assert result["Postcode source"] == "detail_address"
+    assert result["Inferred postcode"] == "BR1 2AB"
+    assert (result["lat"], result["lon"]) == (51.401, 0.011)
+
+
+def test_transform_uses_nearest_when_detail_postcode_mismatches() -> None:
+    detail = {"lat": 51.401, "lng": 0.011, "postcode": "E14 9SS", "outcode": "E14"}
+    result = transform_property(
+        _raw(), StubPostcodeIndex("BR1 2AB"), PC_COORDS, search_outcode="BR1", detail=detail
+    )
+    assert result is not None
+    # Mismatching detail postcode is rejected in favour of the spatial value.
+    assert result["Postcode"] == "BR1 2AB"
+    assert result["Postcode source"] == "detail_coordinates"
+
+
+def test_transform_geocodes_detail_postcode_without_coordinates() -> None:
+    detail = {"lat": None, "lng": None, "postcode": "SW1A 1AA", "outcode": "SW1A"}
+    result = transform_property(
+        _raw(), StubPostcodeIndex(), PC_COORDS, search_outcode="BR1", detail=detail
+    )
+    assert result is not None
+    assert result["Postcode"] == "SW1A 1AA"
+    assert result["Postcode source"] == "detail_address"
+    assert (result["lat"], result["lon"]) == PC_COORDS["SW1A 1AA"]
+
+
+def test_transform_without_detail_falls_back_to_search_outcode() -> None:
+    # No detail, address has no recognizable outcode -> coarse search-outcode centroid.
+    result = transform_property(
+        _raw(address="A street with no postcode"),
+        StubPostcodeIndex(),
+        PC_COORDS,
+        search_outcode="BR1",
+        detail=None,
+    )
+    assert result is not None
+    assert result["Postcode"] == "BR1 2AB"
+    assert result["Postcode source"] == "search_outcode"
+    # No detail page -> no UPRN / house number recovered.
+    assert result["UPRN"] is None
+    assert result["Property number or name"] is None
+
+
+def test_transform_emits_uprn_and_house_numbered_address_from_detail() -> None:
+    detail = {
+        "lat": 51.401,
+        "lng": 0.011,
+        "postcode": "BR1 3CD",
+        "outcode": "BR1",
+        "uprn": "100023461458",
+        "number_or_name": "12",
+        "full_address": "South Street, Bromley BR1",
+    }
+    result = transform_property(
+        _raw(), StubPostcodeIndex("BR1 2AB"), PC_COORDS, search_outcode="BR1", detail=detail
+    )
+    assert result is not None
+    assert result["UPRN"] == "100023461458"
+    assert result["Property number or name"] == "12"
+    # The detail full address replaces the outcode-level card address, and the
+    # house number is prepended for a near-exact Property Register match.
+    assert result["Listing raw address"] == "South Street, Bromley BR1"
+    assert result["Address per Property Register"] == "12, South Street, Bromley"
+
+
+def test_transform_ignores_out_of_england_detail_coords() -> None:
+    detail = {"lat": 10.0, "lng": 10.0, "postcode": "ZZ9 9ZZ", "outcode": "ZZ9"}
+    result = transform_property(
+        _raw(), StubPostcodeIndex("BR1 2AB"), PC_COORDS, search_outcode="BR1", detail=detail
+    )
+    assert result is not None
+    # Bad detail coords are discarded; falls through to the address outcode (BR1).
+    assert result["Postcode source"] == "address_outcode"
+    assert 49 <= result["lat"] <= 56

finder/transform.py

@@ -205,6 +205,41 @@ def extract_full_postcode(text: str | None) -> str | None:
     return normalize_postcode(match.group(1))
 
 
+def extract_outcode(postcode: str | None) -> str | None:
+    """Return the outward code (district) of a UK postcode, e.g. 'SW1A 1AA' → 'SW1A'."""
+    if not postcode:
+        return None
+    normalized = normalize_postcode(postcode)
+    outcode = normalized.split(" ", 1)[0]
+    return outcode or None
+
+
+def resolve_listing_postcode(
+    extracted_postcode: str | None, inferred_postcode: str
+) -> tuple[str, str]:
+    """Pick the authoritative postcode for a listing, returning (postcode, source).
+
+    The address-extracted postcode is more precise than the coordinate-nearest one,
+    but it is only trustworthy when it agrees with the location: a stale, mistyped or
+    well-formed-but-fabricated postcode (e.g. 'ZZ9 9ZZ') would otherwise silently
+    override the spatially-correct value. Since the spatial index only supports
+    nearest-lookup, accept the extracted postcode only when its outcode matches the
+    inferred (coordinate-nearest) postcode's outcode; otherwise fall back to the
+    inferred one, which is always a real, plausibly-correct postcode.
+    """
+    if extracted_postcode and extract_outcode(extracted_postcode) == extract_outcode(
+        inferred_postcode
+    ):
+        return extracted_postcode, "address"
+    if extracted_postcode:
+        log.debug(
+            "Rejecting extracted postcode %s (outcode mismatch with inferred %s)",
+            extracted_postcode,
+            inferred_postcode,
+        )
+    return inferred_postcode, "coordinates"
+
+
 def clean_listing_address(address: str | None) -> str:
     """Remove postcode/outcode suffixes from listing display addresses.
 
@@ -222,10 +257,48 @@ def clean_listing_address(address: str | None) -> str:
     return cleaned.strip(" ,")
 
 
+def build_register_address(
+    raw_address: str | None, number_or_name: str | None = None
+) -> str:
+    """Build a Property Register-style address, prepending the house number/name.
+
+    Listing display addresses are usually street-level ("South Street, Bromley")
+    because the portals hide the exact unit. When a scraper can recover the
+    property's own number or name (e.g. Zoopla detail pages expose
+    ``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address
+    carries the house identifier that the EPC/Price-Paid register addresses also
+    use — turning a fuzzy street match into a near-exact one. Falls back to the
+    plain cleaned address when no number/name is available.
+    """
+    cleaned = clean_listing_address(raw_address)
+    if not number_or_name:
+        return cleaned
+    number_or_name = number_or_name.strip()
+    if not number_or_name:
+        return cleaned
+    # Avoid duplicating a number/name the display address already starts with.
+    if cleaned.lower().startswith(number_or_name.lower()):
+        return cleaned
+    return f"{number_or_name}, {cleaned}" if cleaned else number_or_name
+
+
 def transform_property(
-    prop: dict, outcode: str, pc_index: PostcodeSpatialIndex
+    prop: dict,
+    outcode: str,
+    pc_index: PostcodeSpatialIndex,
+    detail_postcode: str | None = None,
 ) -> dict | None:
-    """Transform a raw Rightmove property dict into our output schema."""
+    """Transform a raw Rightmove property dict into our output schema.
+
+    ``detail_postcode`` is the property's TRUE full postcode recovered from its
+    detail page (see ``rightmove.parse_detail_postcode``); the search API itself
+    only exposes the outcode-level ``displayAddress``. When supplied and it
+    agrees with the coordinate-nearest postcode's outcode, it is preferred over
+    the coordinate guess and recorded with source ``"detail_address"``. A
+    detail postcode whose outcode disagrees with the location is discarded in
+    favour of the spatially-correct coordinate postcode, so a stale or wrong
+    detail value can never silently relocate a listing.
+    """
     loc = prop.get("location")
     if not loc:
         return None
@@ -268,8 +341,25 @@ def transform_property(
         return None
     raw_address = prop.get("displayAddress", "") or ""
     extracted_postcode = extract_full_postcode(raw_address)
-    postcode = extracted_postcode or inferred_postcode
-    postcode_source = "address" if extracted_postcode else "coordinates"
+
+    # Prefer the detail page's true full postcode when it agrees with the
+    # location; otherwise fall back to the (display-address-or-coordinate) logic.
+    detail_full = extract_full_postcode(detail_postcode)
+    if detail_full and extract_outcode(detail_full) == extract_outcode(
+        inferred_postcode
+    ):
+        postcode, postcode_source = detail_full, "detail_address"
+    else:
+        if detail_full:
+            log.debug(
+                "Rejecting Rightmove detail postcode %s (outcode mismatch with "
+                "inferred %s)",
+                detail_full,
+                inferred_postcode,
+            )
+        postcode, postcode_source = resolve_listing_postcode(
+            extracted_postcode, inferred_postcode
+        )
 
     property_url = prop.get("propertyUrl") or ""
     if not isinstance(property_url, str):
@@ -291,6 +381,9 @@ def transform_property(
         "Inferred postcode": inferred_postcode,
         "Listing raw address": raw_address,
         "Address per Property Register": clean_listing_address(raw_address),
+        # Rightmove's displayAddress is street-level; no UPRN/house number.
+        "UPRN": None,
+        "Property number or name": None,
         "Leasehold/Freehold": extract_tenure(prop.get("tenure")),
         "Property type": map_property_type(sub_type),
         "Property sub-type": normalize_sub_type(sub_type),

finder/zoopla.py

@@ -32,16 +32,24 @@ import httpx
 from constants import (
     DATA_DIR,
     DELAY_BETWEEN_PAGES,
+    GLUETUN_API_KEY,
+    GLUETUN_CONTROL_URL,
+    GLUETUN_MAX_ROTATIONS,
+    GLUETUN_PROXY,
     MAX_BEDROOMS,
     PROPERTY_TYPE_MAP,
     ZOOPLA_BASE,
+    ZOOPLA_DETAIL_GOTO_TIMEOUT_MS,
 )
 from spatial import PostcodeSpatialIndex
 from transform import (
-    clean_listing_address,
+    build_register_address,
     extract_full_postcode,
+    extract_outcode,
+    fix_coords,
     normalize_sub_type,
     parse_int_value,
+    resolve_listing_postcode,
     validate_floor_area,
 )
 
@@ -468,27 +476,20 @@ def _challenge_timeout_seconds() -> int:
 # cookies (bound to the previous IP), then reload and re-check the challenge.
 
 
-_GLUETUN_API_KEY = "My8AbvnKhfyFdRhpTVfoTfa5DkAMmg8K"
-
-
 def _gluetun_base_url() -> str:
-    return os.environ.get("GLUETUN_URL", "http://gluetun:8000").rstrip("/")
+    return GLUETUN_CONTROL_URL.rstrip("/")
 
 
 def _gluetun_api_key() -> str | None:
-    return _GLUETUN_API_KEY
+    return GLUETUN_API_KEY
 
 
 def _gluetun_max_rotations() -> int:
-    raw = os.environ.get("GLUETUN_MAX_ROTATIONS", "3")
-    try:
-        value = int(raw)
-    except ValueError as exc:
-        raise ValueError("GLUETUN_MAX_ROTATIONS must be an integer") from exc
-    return max(value, 0)
+    return max(GLUETUN_MAX_ROTATIONS, 0)
 
 
 def _gluetun_client() -> httpx.Client:
+    # Talks to the control server directly (not through the VPN proxy).
     headers = {}
     api_key = _gluetun_api_key()
     if api_key:
@@ -694,10 +695,19 @@ def launch_browser():
     profile_dir.mkdir(parents=True, exist_ok=True)
     _remove_stale_profile_locks(profile_dir)
 
+    # Route the browser through the Gluetun VPN proxy when configured. (geoip
+    # fingerprint alignment is intentionally not enabled: it needs the optional
+    # camoufox[geoip] extra and would spoof to the VPN exit's country, which
+    # fights the en-GB locale unless the exit is in the UK.)
+    proxy_options: dict = {}
+    if GLUETUN_PROXY:
+        proxy_options = {"proxy": {"server": GLUETUN_PROXY}}
+
     log.info(
-        "Launching Camoufox browser for Zoopla (headless=%s, profile=%s)...",
+        "Launching Camoufox browser for Zoopla (headless=%s, profile=%s, proxy=%s)...",
         headless_mode,
         profile_dir,
+        GLUETUN_PROXY or "direct",
     )
     camoufox = Camoufox(
         headless=headless_mode,
@@ -705,6 +715,7 @@ def launch_browser():
         user_data_dir=str(profile_dir),
         locale=["en-GB", "en"],
         enable_cache=True,
+        **proxy_options,
     )
     raw_browser = camoufox.__enter__()
     browser = _ManagedCamoufoxBrowser(camoufox, raw_browser)
@@ -926,13 +937,47 @@ def _paginate(
     page,
     total_results: int,
     max_properties: int | None = None,
+    fetch_detail=None,
+    detail_cap: int = 0,
+    detail_state: dict | None = None,
+    detail_deadline: float | None = None,
 ) -> list[dict]:
     """Extract listings from all pages of search results.
 
     Page 1 is already loaded. For subsequent pages, follow Zoopla's rendered
     next link when present, otherwise advance via the pn=N URL parameter while
-    the advertised result count says more listings remain."""
+    the advertised result count says more listings remain.
+
+    When ``fetch_detail`` is supplied, each listing has its detail page fetched
+    (up to ``detail_cap`` fresh loads per outcode, counted in the shared
+    ``detail_state`` dict, and only until ``detail_deadline``) and the parsed
+    geo stored under ``listing['_detail']`` for ``transform_property``. The
+    detail page is the only source of the listing's UPRN, full street address
+    and precise postcode, so it is fetched even when the search card already
+    pins a full postcode. Cached detail results are always attached but cost
+    neither a cap slot nor a delay."""
+
+    def _maybe_fetch(listing: dict) -> None:
+        if fetch_detail is None or detail_state is None:
+            return
+        url = listing.get("url", "")
+        cached = _detail_cache_key(url) in _detail_cache
+        if not cached:
+            # Fresh loads are bounded by the per-outcode cap and the wall-clock
+            # deadline so detail fetching never starves the SIGALRM budget that
+            # also guards the search pagination for this outcode.
+            if detail_state["fetched"] >= detail_cap:
+                return
+            if detail_deadline is not None and time.monotonic() >= detail_deadline:
+                return
+        listing["_detail"] = fetch_detail(url)
+        if not cached:
+            detail_state["fetched"] += 1
+            time.sleep(DELAY_BETWEEN_PAGES)
+
     all_listings = _extract_listings(page)
+    for listing in all_listings:
+        _maybe_fetch(listing)
     if max_properties is not None and len(all_listings) >= max_properties:
         return all_listings[:max_properties]
 
@@ -984,6 +1029,7 @@ def _paginate(
             if listing["id"] not in seen_ids:
                 seen_ids.add(listing["id"])
                 all_listings.append(listing)
+                _maybe_fetch(listing)
                 new_count += 1
                 if max_properties is not None and len(all_listings) >= max_properties:
                     return all_listings[:max_properties]
@@ -1053,6 +1099,214 @@ def _extract_outcode(text: str) -> str | None:
     return None
 
 
+# ---------------------------------------------------------------------------
+# Detail-page geocoding
+# ---------------------------------------------------------------------------
+#
+# Zoopla search result cards only expose an outcode-level display address (e.g.
+# "South Street, Bromley BR1"); the full postcode and precise coordinates exist
+# only on each listing's detail page (/for-sale/details/{id}/). The detail page
+# is a Next.js App Router route whose React Server Components flight stream
+# embeds the property's own location object, e.g.
+#   "location":{"outcode":"NR29","coordinates":{"latitude":52.716,"longitude":1.614},
+#               "uprn":"10023461458","postalCode":"NR29 4RG",...}
+# plus a twin "address":{"fullAddress":...,"latitude":...,"longitude":...,
+#               "outcode":...,"postcode":...,"uprn":...} feeding the map widgets.
+# Nearby points of interest (stations, schools, EV chargers) and comparable
+# listings carry their own "coordinates" too, but never inside the property's
+# own "location" / "address":{"fullAddress" wrapper — so the wrapper, not a
+# loose coordinates object, is what we anchor on (see parse_detail_geo).
+
+# listingId -> parsed detail dict (or None). Failures are cached too, so a
+# broken listing is not re-fetched within a run (the same listing reappears
+# across overlapping outcode searches).
+_detail_cache: dict[str, dict | None] = {}
+
+_LISTING_ID_RE = re.compile(r"/details/(\d+)/?")
+
+# The property's own location is carried by a `"location":{...}` wrapper and a
+# twin `"address":{"fullAddress":...}` widget object. We anchor on those
+# wrappers (and capture their full object body, which contains exactly one
+# nested object — `coordinates`) rather than scanning for loose coordinate
+# objects: nearby points of interest (stations/schools/EV chargers) and
+# comparable/"similar" listings also embed coordinates, but never inside the
+# property's own `"location"` / `"address":{"fullAddress"` wrapper, so the
+# wrapper is the discriminator. Field order and an optional `uprn` are tolerated.
+_DETAIL_LOCATION_RE = re.compile(r'"location":\{((?:[^{}]|\{[^{}]*\})*)\}')
+_DETAIL_ADDRESS_RE = re.compile(r'"address":\{"fullAddress":"([^"]*)"((?:[^{}]|\{[^{}]*\})*)\}')
+_DETAIL_COORDS_IN_BODY_RE = re.compile(
+    r'"coordinates":\{"latitude":(-?\d+\.\d+),"longitude":(-?\d+\.\d+)\}'
+)
+_DETAIL_LATLNG_IN_BODY_RE = re.compile(
+    r'"latitude":(-?\d+\.\d+),"longitude":(-?\d+\.\d+)'
+)
+_DETAIL_OUTCODE_IN_BODY_RE = re.compile(r'"outcode":"([A-Z0-9]+)"')
+# The location object spells it "postalCode"; the address twin uses "postcode".
+_DETAIL_POSTCODE_IN_BODY_RE = re.compile(r'"(?:postalCode|postcode)":"([A-Z0-9 ]+)"')
+# The UPRN (Unique Property Reference Number) appears in both the location and
+# address objects and is the linchpin for an exact listing->EPC join (EPC open
+# data is ~99% UPRN-keyed). propertyNumberOrName carries the house number/name
+# (e.g. "12", "Martham Mill") only in the location object.
+_DETAIL_UPRN_IN_BODY_RE = re.compile(r'"uprn":"(\d+)"')
+_DETAIL_NUMBER_OR_NAME_IN_BODY_RE = re.compile(r'"propertyNumberOrName":"([^"]*)"')
+
+
+def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | None:
+    """Extract the property's own coordinates/postcode from a Zoopla detail page.
+
+    Pure and browser-free: the live browser only produces the HTML string
+    (``page.content()``); this does the parsing so it is unit-testable.
+
+    Returns ``{"lat", "lng", "postcode", "outcode", "source", "uprn",
+    "number_or_name", "full_address"}`` (every field except the coordinates may
+    be ``None``) or ``None`` when no property location wrapper is found. The
+    ``uprn`` enables an exact listing->EPC join; ``number_or_name`` (house
+    number/name) and ``full_address`` give a register-style address for the
+    Price Paid join.
+    Coordinates are bounds-checked to England and a postcode is kept only when
+    it agrees with its own object's outcode. ``search_outcode``, when given, is
+    used only as a tie-break to pick the right ``location`` object on pages that
+    also embed comparable listings. See module docstring for the data model."""
+    if not html:
+        return None
+
+    # RSC flight strings are embedded as escaped JS string literals, so quotes
+    # and slashes arrive escaped; normalize them so the regexes match.
+    buf = html.replace('\\"', '"').replace("\\u002F", "/").replace("\\/", "/")
+
+    def in_england(lat: float, lng: float) -> tuple[float, float] | None:
+        lat, lng = fix_coords(lat, lng)
+        if 49 <= lat <= 56 and -7 <= lng <= 2:
+            return lat, lng
+        return None
+
+    def build(body: str, coords, source: str, full_address: str | None = None) -> dict:
+        # outcode and postcode are read from the SAME object body as the coords,
+        # so the postcode is self-consistent; drop it only if it somehow isn't.
+        outcode_match = _DETAIL_OUTCODE_IN_BODY_RE.search(body)
+        outcode = outcode_match.group(1) if outcode_match else None
+        postcode_match = _DETAIL_POSTCODE_IN_BODY_RE.search(body)
+        postcode = extract_full_postcode(postcode_match.group(1)) if postcode_match else None
+        if postcode and outcode and extract_outcode(postcode) != outcode.upper():
+            postcode = None
+        uprn_match = _DETAIL_UPRN_IN_BODY_RE.search(body)
+        number_match = _DETAIL_NUMBER_OR_NAME_IN_BODY_RE.search(body)
+        number_or_name = number_match.group(1).strip() if number_match else None
+        return {
+            "lat": coords[0],
+            "lng": coords[1],
+            "postcode": postcode,
+            "outcode": outcode,
+            "source": source,
+            "uprn": uprn_match.group(1) if uprn_match else None,
+            "number_or_name": number_or_name or None,
+            "full_address": full_address,
+        }
+
+    def attach_full_address(result: dict | None) -> dict | None:
+        # The house-numbered street address lives in the `address` map-widget
+        # twin, not the `location` wrapper we anchor coordinates on. Pull it from
+        # the twin that shares this property's uprn; when there is no uprn to
+        # disambiguate, fall back to the first twin (document order = primary
+        # listing), but never guess a twin when a uprn exists and none matches —
+        # that would risk grabbing a comparable listing's address.
+        if result is None or result.get("full_address"):
+            return result
+        target = result.get("uprn")
+        first = None
+        for match in _DETAIL_ADDRESS_RE.finditer(buf):
+            full_address = match.group(1) or None
+            if full_address is None:
+                continue
+            if first is None:
+                first = full_address
+            uprn_match = _DETAIL_UPRN_IN_BODY_RE.search(match.group(2))
+            if target and uprn_match and uprn_match.group(1) == target:
+                result["full_address"] = full_address
+                return result
+        if target is None:
+            result["full_address"] = first
+        return result
+
+    # Strategy 1 — the property's own `location` wrapper (authoritative). Take
+    # the first match (the primary listing precedes any comparables in the
+    # flight stream), but prefer one whose outcode matches the searched outcode.
+    first_location = None
+    for match in _DETAIL_LOCATION_RE.finditer(buf):
+        body = match.group(1)
+        coords_match = _DETAIL_COORDS_IN_BODY_RE.search(body)
+        if not coords_match:
+            continue
+        coords = in_england(float(coords_match.group(1)), float(coords_match.group(2)))
+        if not coords:
+            continue
+        candidate = build(body, coords, "detail_location")
+        if first_location is None:
+            first_location = candidate
+        if (
+            search_outcode
+            and candidate["outcode"]
+            and candidate["outcode"].upper() == search_outcode.upper()
+        ):
+            return attach_full_address(candidate)
+    if first_location is not None:
+        return attach_full_address(first_location)
+
+    # Strategy 2 — the `address` map-widget twin (same coordinates, backup).
+    for match in _DETAIL_ADDRESS_RE.finditer(buf):
+        full_address = match.group(1) or None
+        body = match.group(2)
+        latlng_match = _DETAIL_LATLNG_IN_BODY_RE.search(body)
+        if not latlng_match:
+            continue
+        coords = in_england(float(latlng_match.group(1)), float(latlng_match.group(2)))
+        if coords:
+            return build(body, coords, "detail_address_obj", full_address=full_address)
+
+    return None
+
+
+def _detail_cache_key(listing_url: str) -> str:
+    """Cache key for a listing detail page — its numeric id when present."""
+    id_match = _LISTING_ID_RE.search(listing_url)
+    return id_match.group(1) if id_match else listing_url
+
+
+def _fetch_listing_detail(
+    detail_page,
+    listing_url: str,
+    search_outcode: str | None = None,
+) -> dict | None:
+    """Load a listing detail page and return its parsed geo dict (or None).
+
+    Results (including failures) are cached by listingId. Ordinary navigation
+    and extraction errors are swallowed so the caller can fall back to
+    outcode-level resolution, but TurnstileError is allowed to propagate so the
+    scraper's "Cloudflare ends the run" contract still holds. The goto timeout
+    is kept short so one slow detail page can't eat the per-outcode budget."""
+    cache_key = _detail_cache_key(listing_url)
+    if cache_key in _detail_cache:
+        return _detail_cache[cache_key]
+
+    url = listing_url if listing_url.startswith("http") else ZOOPLA_BASE + listing_url
+    result: dict | None = None
+    try:
+        detail_page.goto(
+            url, wait_until="domcontentloaded", timeout=ZOOPLA_DETAIL_GOTO_TIMEOUT_MS
+        )
+        _ensure_not_challenged(detail_page)
+        html = detail_page.content()
+        result = parse_detail_geo(html, search_outcode=search_outcode)
+    except TurnstileError:
+        raise
+    except Exception as exc:
+        log.debug("Zoopla detail fetch failed %s: %s", url, _exception_detail(exc))
+        result = None
+
+    _detail_cache[cache_key] = result
+    return result
+
+
 def _map_property_type(raw_type: str | None) -> str:
     """Map Zoopla property type text to canonical type."""
     if not raw_type:
@@ -1109,28 +1363,64 @@ def transform_property(
     pc_index: PostcodeSpatialIndex,
     pc_coords: dict[str, tuple[float, float]],
     search_outcode: str | None = None,
+    detail: dict | None = None,
 ) -> dict | None:
     """Transform a raw Zoopla listing dict into the standard output schema.
 
-    Zoopla search cards do not include coordinates, so we resolve lat/lng
-    from postcodes extracted from the address text."""
+    Zoopla search cards only expose an outcode-level address, so precise
+    location comes from the listing's detail page (see ``parse_detail_geo`` /
+    ``_fetch_listing_detail``), passed in as ``detail``. When detail-page
+    coordinates are available we resolve the nearest postcode via the spatial
+    index — mirroring rightmove/onthemarket — and only fall back to the coarse
+    outcode centroid when no detail location could be obtained."""
     price = parse_int_value(raw.get("price")) or 0
 
     address = raw.get("address", "") or ""
 
-    # Resolve postcode and coordinates from address
     extracted_postcode = extract_full_postcode(address)
-    postcode = extracted_postcode
-    postcode_source = "address" if extracted_postcode else None
+    detail = detail or {}
+    detail_postcode = extract_full_postcode(detail.get("postcode"))
+    # Detail-page address fields: the UPRN keys an exact EPC join, and the
+    # full street address / house number-or-name beat the outcode-level card
+    # address for the Price-Paid join. All three are absent unless the detail
+    # page was fetched, so every consumer must tolerate None.
+    detail_uprn = detail.get("uprn") or None
+    detail_full_address = detail.get("full_address") or None
+    detail_number_or_name = detail.get("number_or_name") or None
+
+    postcode = postcode_source = inferred_postcode = None
     lat = lng = None
 
-    if postcode:
-        coords = pc_coords.get(postcode)
-        if coords:
-            lat, lng = coords
+    # (A) Best: detail-page coordinates -> nearest postcode (authoritative).
+    detail_lat, detail_lng = detail.get("lat"), detail.get("lng")
+    if detail_lat is not None and detail_lng is not None:
+        fixed_lat, fixed_lng = fix_coords(detail_lat, detail_lng)
+        if 49 <= fixed_lat <= 56 and -7 <= fixed_lng <= 2:
+            nearest = pc_index.nearest(fixed_lat, fixed_lng)
+            if nearest:
+                lat, lng, inferred_postcode = fixed_lat, fixed_lng, nearest
+                candidate = detail_postcode or extracted_postcode
+                postcode, resolved_source = resolve_listing_postcode(candidate, nearest)
+                postcode_source = (
+                    "detail_address"
+                    if resolved_source == "address"
+                    else "detail_coordinates"
+                )
 
+    # (B) Detail-page postcode without usable coordinates -> geocode it.
+    if lat is None and detail_postcode and detail_postcode in pc_coords:
+        lat, lng = pc_coords[detail_postcode]
+        postcode = inferred_postcode = detail_postcode
+        postcode_source = "detail_address"
+
+    # (C) Full postcode in the search-card address -> geocode it.
+    if lat is None and extracted_postcode and extracted_postcode in pc_coords:
+        lat, lng = pc_coords[extracted_postcode]
+        postcode = extracted_postcode
+        postcode_source = "address"
+
+    # (D) Last resort: coarse outcode-level centroid (loses per-listing precision).
     if lat is None:
-        # Try outcode-level fallback from address text
         addr_outcode = _extract_outcode(address)
         if addr_outcode:
             result = _resolve_outcode_coords(addr_outcode, pc_coords)
@@ -1138,7 +1428,6 @@ def transform_property(
                 postcode, lat, lng = result
                 postcode_source = "address_outcode"
 
-    # Final fallback: use the outcode we know we're searching
     if lat is None and search_outcode:
         result = _resolve_outcode_coords(search_outcode, pc_coords)
         if result:
@@ -1188,9 +1477,17 @@ def transform_property(
         "Postcode": postcode,
         "Postcode source": postcode_source or "unknown",
         "Extracted postcode": extracted_postcode,
-        "Inferred postcode": postcode if postcode_source != "address" else None,
-        "Listing raw address": address,
-        "Address per Property Register": clean_listing_address(address),
+        "Inferred postcode": (
+            inferred_postcode
+            if inferred_postcode is not None
+            else (postcode if postcode_source != "address" else None)
+        ),
+        "Listing raw address": detail_full_address or address,
+        "Address per Property Register": build_register_address(
+            detail_full_address or address, detail_number_or_name
+        ),
+        "UPRN": detail_uprn,
+        "Property number or name": detail_number_or_name,
         "Leasehold/Freehold": raw.get("tenure") or None,
         "Property type": _map_property_type(raw.get("property_type")),
         "Property sub-type": normalize_sub_type(raw.get("property_type")),
@@ -1215,6 +1512,9 @@ def search_outcode(
     pc_index: PostcodeSpatialIndex,
     pc_coords: dict[str, tuple[float, float]],
     max_properties: int | None = None,
+    detail_page=None,
+    detail_cap: int = 0,
+    detail_budget_seconds: float | None = None,
 ) -> tuple[list[dict], str | None]:
     """Search Zoopla for properties in one outcode.
 
@@ -1222,6 +1522,12 @@ def search_outcode(
     search flow, extracts listings from rendered DOM, and transforms to the
     standard output schema.
 
+    When ``detail_page`` (a second browser tab) and a positive ``detail_cap``
+    are supplied, up to ``detail_cap`` listings per outcode have their detail
+    page fetched for a precise postcode (see ``_fetch_listing_detail``).
+    ``detail_budget_seconds`` caps the wall-clock time spent fetching details so
+    the per-outcode timeout that also guards search pagination is never starved.
+
     Returns (properties, search_url).
 
     Raises TurnstileError if Cloudflare blocks us mid-session.
@@ -1231,12 +1537,25 @@ def search_outcode(
 
     total_results = _get_result_count(page)
 
+    fetch_detail = None
+    detail_deadline = None
+    if detail_page is not None and detail_cap > 0:
+        fetch_detail = lambda url: _fetch_listing_detail(  # noqa: E731
+            detail_page, url, search_outcode=outcode
+        )
+        if detail_budget_seconds is not None:
+            detail_deadline = time.monotonic() + detail_budget_seconds
+
     # Always try extraction even if result count is 0 — the count regex may
     # not match Zoopla's current text format, but listings may still be in DOM
     raw_listings = _paginate(
         page,
         total_results,
         max_properties=max_properties,
+        fetch_detail=fetch_detail,
+        detail_cap=detail_cap,
+        detail_state={"fetched": 0},
+        detail_deadline=detail_deadline,
     )
     if not raw_listings:
         if total_results > 0:
@@ -1252,7 +1571,11 @@ def search_outcode(
     for raw in raw_listings:
         try:
             transformed = transform_property(
-                raw, pc_index, pc_coords, search_outcode=outcode
+                raw,
+                pc_index,
+                pc_coords,
+                search_outcode=outcode,
+                detail=raw.get("_detail"),
             )
         except Exception as exc:
             log.warning(

finder/zoopla_flaresolverr.py

@@ -0,0 +1,164 @@
+"""Zoopla scraping via FlareSolverr (no browser/VNC needed).
+
+FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which
+still contains the React Server Components flight stream — so the existing pure
+parsers work unchanged:
+  - the search page yields the outcode's listing detail URLs, and
+  - each detail page's flight stream carries the property's location object
+    (postcode + coordinates) that ``parse_detail_geo`` extracts, plus the
+    listing fields (price/beds/baths/tenure/floor area) parsed here.
+
+Verified live (2026-05-30) against Zoopla through the Gluetun VPN: a warm
+FlareSolverr session solves the SW9 search + detail pages and the flight data
+is present (e.g. detail 73326946 -> SW9 0HD @ 51.477238,-0.116819).
+
+This is selected by constants.ZOOPLA_FETCHER == "flaresolverr"; the Camoufox
+path in zoopla.py remains for ZOOPLA_FETCHER == "camoufox".
+"""
+
+import logging
+import re
+import time
+
+from constants import DELAY_BETWEEN_PAGES, ZOOPLA_BASE
+from flaresolverr import FlareSolverrError, FlareSolverrSession
+from spatial import PostcodeSpatialIndex
+from zoopla import _url_with_page, parse_detail_geo, transform_property
+
+log = logging.getLogger("zoopla")
+
+# Safety bound on how many search-result pages to walk per outcode.
+_MAX_SERP_PAGES = 60
+
+_DETAIL_PATH_RE = re.compile(r"/(?:for-sale|new-homes)/details/\d+/")
+_LISTING_ID_RE = re.compile(r"/details/(\d+)/")
+
+
+def _int(pattern: str, buf: str) -> int | None:
+    match = re.search(pattern, buf)
+    return int(match.group(1)) if match else None
+
+
+def parse_detail_listing(html: str) -> dict:
+    """Extract the non-location listing fields from a Zoopla detail page.
+
+    Mirrors the fields the Camoufox SERP-card extractor produced, read from the
+    detail page's flight stream (validated against real Zoopla detail HTML).
+    All fields are best-effort; missing ones default to None so a listing with
+    a known location is still emitted."""
+    buf = html.replace('\\"', '"').replace("\\/", "/")
+
+    price = _int(r'"internalValue":(\d+)', buf)
+    if price is None:
+        price = _int(r'"priceUnformatted":(\d+)', buf)
+
+    tenure_match = re.search(r'"tenure":"([a-zA-Z]+)"', buf)
+    tenure = tenure_match.group(1).title() if tenure_match else None
+
+    # Address + property type come from the page <title>, e.g.
+    # "Caldwell Street, Stockwell SW9, 4 bed property for sale, £995,000 - Zoopla"
+    address = None
+    property_type = None
+    title_match = re.search(r'"children":"([^"]*? for sale[^"]*?)"', buf)
+    if title_match:
+        title = title_match.group(1)
+        addr_match = re.match(r"(.+?),\s*\d+\s*bed", title)
+        if addr_match:
+            address = addr_match.group(1).strip()
+        type_match = re.search(r"\d+\s*bed\s+([\w\s-]+?)\s+for sale", title)
+        if type_match:
+            property_type = type_match.group(1).strip()
+    explicit_type = re.search(r'"propertyType":"([^"]+)"', buf)
+    if explicit_type:
+        property_type = explicit_type.group(1)
+
+    return {
+        "price": price,
+        "beds": _int(r'"numBedrooms":(\d+)', buf),
+        "baths": _int(r'"numBaths":(\d+)', buf),
+        "receptions": _int(r'"numLivingRooms":(\d+)', buf),
+        "floor_area_sqft": _int(r'"sizeSqft":(\d+)', buf),
+        "tenure": tenure,
+        "property_type": property_type,
+        "address": address,
+    }
+
+
+def _enumerate_detail_paths(fs: FlareSolverrSession, outcode: str, limit: int | None) -> list[str]:
+    """Walk the outcode's search-result pages and collect listing detail paths."""
+    base = f"{ZOOPLA_BASE}/for-sale/property/{outcode.lower()}/?q={outcode}&search_source=home"
+    seen: list[str] = []
+    seen_ids: set[str] = set()
+    for page_num in range(1, _MAX_SERP_PAGES + 1):
+        url = base if page_num == 1 else _url_with_page(base, page_num)
+        html = fs.get(url)
+        new = 0
+        for path in _DETAIL_PATH_RE.findall(html):
+            id_match = _LISTING_ID_RE.search(path)
+            listing_id = id_match.group(1) if id_match else path
+            if listing_id in seen_ids:
+                continue
+            seen_ids.add(listing_id)
+            seen.append(path)
+            new += 1
+            if limit is not None and len(seen) >= limit:
+                return seen
+        if new == 0:
+            break
+        time.sleep(DELAY_BETWEEN_PAGES)
+    return seen
+
+
+def search_outcode(
+    outcode: str,
+    pc_index: PostcodeSpatialIndex,
+    pc_coords: dict[str, tuple[float, float]],
+    fs: FlareSolverrSession,
+    max_properties: int | None = None,
+    detail_cap: int = 0,
+    detail_budget_seconds: float | None = None,
+) -> tuple[list[dict], str | None]:
+    """Scrape one outcode via FlareSolverr. Returns (properties, search_url).
+
+    Every listing's detail page is fetched (that is where the postcode lives),
+    so the effective listing count is bounded by both ``max_properties`` and
+    ``detail_cap``; ``detail_budget_seconds`` caps wall-clock time on details."""
+    limit = detail_cap if detail_cap and detail_cap > 0 else None
+    if max_properties is not None:
+        limit = max_properties if limit is None else min(limit, max_properties)
+
+    base = f"{ZOOPLA_BASE}/for-sale/property/{outcode.lower()}/?q={outcode}&search_source=home"
+    paths = _enumerate_detail_paths(fs, outcode, limit)
+    if not paths:
+        return [], base
+
+    deadline = (time.monotonic() + detail_budget_seconds) if detail_budget_seconds else None
+    properties: list[dict] = []
+    dropped = 0
+    for path in paths:
+        if deadline is not None and time.monotonic() >= deadline:
+            log.info("Zoopla %s: detail-fetch budget reached after %d", outcode, len(properties))
+            break
+        id_match = _LISTING_ID_RE.search(path)
+        listing_id = id_match.group(1) if id_match else path
+        try:
+            html = fs.get(ZOOPLA_BASE + path)
+            geo = parse_detail_geo(html, search_outcode=outcode)
+            raw = {"id": listing_id, "url": path, **parse_detail_listing(html)}
+            prop = transform_property(
+                raw, pc_index, pc_coords, search_outcode=outcode, detail=geo
+            )
+        except FlareSolverrError as exc:
+            log.warning("Zoopla %s detail %s fetch failed: %s", outcode, listing_id, exc)
+            prop = None
+        except Exception as exc:  # noqa: BLE001 - never let one listing kill the outcode
+            log.warning("Zoopla %s detail %s transform failed: %s", outcode, listing_id, exc)
+            prop = None
+        if prop:
+            properties.append(prop)
+        else:
+            dropped += 1
+        time.sleep(DELAY_BETWEEN_PAGES)
+
+    log.info("Zoopla %s: %d listings (%d dropped)", outcode, len(properties), dropped)
+    return properties, base

frontend/src/App.tsx

@@ -81,6 +81,15 @@ function isProtectedPage(page: Page): boolean {
   return page === 'account' || page === 'saved';
 }
 
+function isSharedDashboardUrl(): boolean {
+  const share = new URLSearchParams(window.location.search).get('share');
+  return !!share && /^[a-z0-9]{1,20}$/i.test(share);
+}
+
+function isAuthRequiredRoute(page: Page): boolean {
+  return isProtectedPage(page) || (page === 'dashboard' && !isSharedDashboardUrl());
+}
+
 function buildPageUrl(page: Page, inviteCode?: string, search = '', hash = ''): string {
   const normalizedHash = normalizeHash(hash);
   return `${pageToPath(page, inviteCode)}${search}${normalizedHash ? `#${normalizedHash}` : ''}`;
@@ -235,6 +244,7 @@ export default function App() {
   const postAuthCheckoutReturnPathRef = useRef<string | null>(null);
   const authCompletedRef = useRef(false);
   const [licenseSuccessStatus, setLicenseSuccessStatus] = useState<LicenseSuccessStatus>('hidden');
+  const [dashboardReady, setDashboardReady] = useState(false);
 
   // Keep a ref to the latest refreshAuth so the mount-only startup effect always
   // calls the current implementation without re-running when the callback identity changes.
@@ -266,7 +276,7 @@ export default function App() {
     if (!completed) {
       setPostAuthIntent(null);
       postAuthCheckoutReturnPathRef.current = null;
-      if (isProtectedPage(activePageRef.current)) {
+      if (isAuthRequiredRoute(activePageRef.current)) {
         window.history.replaceState({ page: 'home', hash: '' }, '', '/');
         setRouteHash('');
         setActivePage('home');
@@ -517,7 +527,10 @@ export default function App() {
     }
   }, [activePage, fetchSearches]);
 
-  const isAuthRequiredPage = activePage === 'account' || activePage === 'saved';
+  const isAuthRequiredPage =
+    activePage === 'account' ||
+    activePage === 'saved' ||
+    (activePage === 'dashboard' && !mapUrlState.share);
   useEffect(() => {
     if (authLoading) return;
     if (isAuthRequiredPage && !user) {
@@ -530,6 +543,13 @@ export default function App() {
 
   const [exportState, setExportState] = useState<ExportState | null>(null);
 
+  useEffect(() => {
+    if (activePage !== 'dashboard' || !user) {
+      setDashboardReady(false);
+      setExportState(null);
+    }
+  }, [activePage, user]);
+
   if ((isScreenshotMode || isOgMode) && inviteCode) {
     return (
       <Suspense fallback={<PageFallback />}>
@@ -584,8 +604,9 @@ export default function App() {
         onPageChange={navigateTo}
         theme={theme}
         onToggleTheme={toggleTheme}
-        exportState={activePage === 'dashboard' ? exportState : null}
+        exportState={activePage === 'dashboard' && user ? exportState : null}
         dashboardParams={activePage === 'dashboard' ? dashboardParams : ''}
+        dashboardActionsDisabled={activePage === 'dashboard' && !dashboardReady}
         onSaveSearch={
           activePage === 'dashboard' && user
             ? editingSearch
@@ -675,6 +696,7 @@ export default function App() {
             onNavigateTo={navigateTo}
             onExportStateChange={setExportState}
             onDashboardParamsChange={setDashboardParams}
+            onDashboardReadyChange={setDashboardReady}
             isMobile={isMobile}
             initialTravelTime={mapUrlState.travelTime}
             initialPostcode={mapUrlState.postcode}

frontend/src/components/account/AccountPage.tsx

@@ -461,6 +461,24 @@ interface ShareLinkListItem {
   created: string;
 }
 
+function latestPendingInviteUrls(invites: InviteListItem[]): Record<string, string> {
+  const latestByType: Record<string, { url: string; createdMs: number }> = {};
+
+  for (const invite of invites) {
+    if (invite.used || !invite.url) continue;
+
+    const createdMs = Date.parse(invite.created) || 0;
+    const existing = latestByType[invite.invite_type];
+    if (!existing || createdMs > existing.createdMs) {
+      latestByType[invite.invite_type] = { url: invite.url, createdMs };
+    }
+  }
+
+  return Object.fromEntries(
+    Object.entries(latestByType).map(([type, invite]) => [type, invite.url])
+  );
+}
+
 function InviteTable({
   invites,
   loading,
@@ -673,7 +691,16 @@ function InviteSection({ user }: { user: AuthUser }) {
       const res = await fetch(apiUrl('invites'), authHeaders());
       assertOk(res, 'Fetch invites');
       const data = await res.json();
-      setInviteHistory(data.invites);
+      const invites: InviteListItem[] = Array.isArray(data.invites) ? data.invites : [];
+      setInviteHistory(invites);
+      const pendingInviteUrls = latestPendingInviteUrls(invites);
+      setInviteUrl((prev) => {
+        const next = { ...prev };
+        for (const [type, url] of Object.entries(pendingInviteUrls)) {
+          if (!next[type]) next[type] = url;
+        }
+        return next;
+      });
     } catch {
       // Silent — non-critical
     } finally {

frontend/src/components/map/LocationSearch.test.tsx

@@ -8,8 +8,11 @@ const RECENT_SEARCHES_STORAGE_KEY = 'perfect-postcode.locationSearch.recent';
 
 vi.mock('react-i18next', () => ({
   useTranslation: () => ({
-    t: (key: string) =>
-      key === 'locationSearch.placeholder' ? 'Search places or postcodes...' : key,
+    t: (key: string) => {
+      if (key === 'locationSearch.placeholder') return 'Search places or postcodes...';
+      if (key === 'locationSearch.noResults') return 'No matching places or postcodes';
+      return key;
+    },
   }),
 }));
 
@@ -226,6 +229,170 @@ describe('LocationSearch', () => {
     );
   });
 
+  it('selects the first place suggestion with Enter when none is highlighted', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn((input: string | URL | Request) => {
+        const url = new URL(String(input), 'http://localhost');
+        if (url.pathname === '/api/places') {
+          return Promise.resolve(
+            jsonResponse({
+              places: [
+                {
+                  type: 'place',
+                  name: 'London',
+                  slug: 'london',
+                  place_type: 'city',
+                  lat: 51.5074,
+                  lon: -0.1278,
+                },
+              ],
+              postcodes: [],
+              addresses: [],
+            })
+          );
+        }
+        if (url.pathname === '/api/nearest-postcode') {
+          return Promise.resolve(
+            jsonResponse({
+              postcode: 'SW1A 1AA',
+              latitude: 51.501,
+              longitude: -0.141,
+              geometry: postcodeGeometry,
+            })
+          );
+        }
+        return Promise.resolve(new Response(null, { status: 404 }));
+      })
+    );
+
+    const onFlyTo = vi.fn();
+    const onLocationSearched = vi.fn();
+    render(<LocationSearch onFlyTo={onFlyTo} onLocationSearched={onLocationSearched} />);
+
+    const input = screen.getByRole('textbox');
+    fireEvent.change(input, { target: { value: 'London' } });
+
+    await screen.findByRole('button', { name: 'London' });
+    fireEvent.keyDown(input, { key: 'Enter' });
+
+    await waitFor(() => {
+      expect(onLocationSearched).toHaveBeenCalledTimes(1);
+    });
+    expect(onFlyTo).toHaveBeenCalledWith(51.5074, -0.1278, 10);
+    expect(onLocationSearched).toHaveBeenCalledWith({
+      postcode: 'SW1A 1AA',
+      geometry: postcodeGeometry,
+      latitude: 51.501,
+      longitude: -0.141,
+      zoom: 10,
+      markerLatitude: 51.5074,
+      markerLongitude: -0.1278,
+    });
+  });
+
+  it('preserves the server unified ordering and sends the viewport centre', async () => {
+    const fetchMock = vi.fn((input: string | URL | Request) => {
+      const url = new URL(String(input), 'http://localhost');
+      if (url.pathname === '/api/places') {
+        return Promise.resolve(
+          jsonResponse({
+            places: [],
+            postcodes: [],
+            addresses: [],
+            // Intentionally out of "bucket" order: an address outranks a place. The hook must
+            // render them in this server order rather than re-bucketing or re-filtering.
+            results: [
+              {
+                type: 'address',
+                address: '1 High Street',
+                postcode: 'CR0 1AA',
+                lat: 51.37,
+                lon: -0.1,
+                score: 930,
+              },
+              {
+                type: 'place',
+                name: 'Croydon',
+                slug: 'croydon',
+                place_type: 'town',
+                lat: 51.37,
+                lon: -0.1,
+                score: 880,
+              },
+            ],
+          })
+        );
+      }
+      return Promise.resolve(new Response(null, { status: 404 }));
+    });
+    vi.stubGlobal('fetch', fetchMock);
+
+    render(
+      <LocationSearch
+        onFlyTo={vi.fn()}
+        onLocationSearched={vi.fn()}
+        getViewportCenter={() => ({ lat: 51.5, lng: -0.12 })}
+      />
+    );
+
+    fireEvent.change(screen.getByRole('textbox'), { target: { value: 'high street' } });
+
+    // The address (first in the server list) renders before the place, unfiltered.
+    const firstResult = await screen.findByText('1 High Street');
+    const place = screen.getByText('Croydon');
+    expect(
+      firstResult.compareDocumentPosition(place) & Node.DOCUMENT_POSITION_FOLLOWING
+    ).toBeTruthy();
+
+    const placesCall = fetchMock.mock.calls.find(([input]) =>
+      String(input).includes('/api/places')
+    );
+    const calledUrl = new URL(String(placesCall?.[0]), 'http://localhost');
+    expect(calledUrl.searchParams.get('lat')).toBe('51.5');
+    expect(calledUrl.searchParams.get('lng')).toBe('-0.12');
+  });
+
+  it('shows an empty state for invalid place queries (legacy server, no results key)', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() =>
+        Promise.resolve(
+          jsonResponse({
+            places: [],
+            postcodes: [],
+            addresses: [],
+          })
+        )
+      )
+    );
+
+    render(<LocationSearch onFlyTo={vi.fn()} onLocationSearched={vi.fn()} />);
+
+    fireEvent.change(screen.getByRole('textbox'), { target: { value: '!!!!zzzzzz!!!!' } });
+
+    await waitFor(() => {
+      expect(screen.getByText('No matching places or postcodes')).toBeTruthy();
+    });
+  });
+
+  it('shows the empty state when the new server returns an empty results array', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(() =>
+        Promise.resolve(jsonResponse({ places: [], postcodes: [], addresses: [], results: [] }))
+      )
+    );
+
+    render(<LocationSearch onFlyTo={vi.fn()} onLocationSearched={vi.fn()} />);
+
+    fireEvent.change(screen.getByRole('textbox'), { target: { value: 'zzzznowhere' } });
+
+    await waitFor(() => {
+      expect(screen.getByText('No matching places or postcodes')).toBeTruthy();
+    });
+  });
+
   it('keeps only the three most recent local searches', async () => {
     vi.stubGlobal(
       'fetch',

frontend/src/components/map/LocationSearch.tsx

@@ -4,7 +4,11 @@ import type { MapFlyToOptions, PostcodeGeometry } from '../../types';
 import { authHeaders, isAbortError } from '../../lib/api';
 import { POSTCODE_SEARCH_ZOOM } from '../../lib/consts';
 import { useIsMobile } from '../../hooks/useIsMobile';
-import { useLocationSearch, type SearchResult } from '../../hooks/useLocationSearch';
+import {
+  useLocationSearch,
+  type SearchResult,
+  type ViewportCenter,
+} from '../../hooks/useLocationSearch';
 import { PlaceSearchInput } from '../ui/PlaceSearchInput';
 import { LocateIcon } from '../ui/icons/LocateIcon';
 import { SearchIcon } from '../ui/icons/SearchIcon';
@@ -44,6 +48,12 @@ const ZOOM_FOR_TYPE: Record<string, number> = {
   locality: 14,
   hamlet: 15,
   isolated_dwelling: 16,
+  street: 16,
+  university: 15,
+  park: 15,
+  attraction: 16,
+  hospital: 16,
+  retail: 15,
 };
 
 const DEV_CURRENT_LOCATION = {
@@ -56,6 +66,7 @@ export default function LocationSearch({
   onLocationSearched,
   onCurrentLocationFound,
   onMouseEnter,
+  getViewportCenter,
   className = '',
   inputClassName,
 }: {
@@ -63,11 +74,13 @@ export default function LocationSearch({
   onLocationSearched?: (postcode: SearchedLocation | null) => void;
   onCurrentLocationFound?: (lat: number, lng: number) => void;
   onMouseEnter?: () => void;
+  /** Returns the current map centre so search ranking can bias toward the visible area. */
+  getViewportCenter?: () => ViewportCenter | null;
   className?: string;
   inputClassName?: string;
 }) {
   const { t } = useTranslation();
-  const search = useLocationSearch();
+  const search = useLocationSearch(undefined, getViewportCenter);
   const [error, setError] = useState<string | null>(null);
   const [loading, setLoading] = useState(false);
   const [expanded, setExpanded] = useState(false);
@@ -333,6 +346,8 @@ export default function LocationSearch({
           onSelect={selectResult}
           loading={loading}
           placeholder={t('locationSearch.placeholder')}
+          ariaLabel={t('locationSearch.searchLabel')}
+          name="location-search"
           size="sm"
           inputClassName={
             inputClassName ??

frontend/src/components/map/Map.tsx

@@ -606,12 +606,13 @@ function OverlayTileLayers({
   const showTrees = activeOverlays.has('trees-outside-woodlands');
   const showPropertyBorders = activeOverlays.has('property-borders');
 
-  // Restrict the heatmap to the selected crime types. When every type is
-  // selected we omit the filter entirely so all features contribute.
-  const crimeFilter =
-    activeCrimeTypes.size >= CRIME_TYPE_VALUES.length
-      ? undefined
-      : ['in', ['get', 'crime_type'], ['literal', Array.from(activeCrimeTypes)]];
+  // Restrict the heatmap to the selected crime types. This must always be a
+  // concrete expression: passing `filter={undefined}` makes react-map-gl call
+  // map.addLayer({filter: undefined}), which MapLibre rejects at validation
+  // ("filter: array expected, undefined found"), so the layer is never created
+  // and the heatmap stays blank until a later setFilter call. An `in` over the
+  // selected types matches everything when all 14 are selected.
+  const crimeFilter = ['in', ['get', 'crime_type'], ['literal', Array.from(activeCrimeTypes)]];
 
   return (
     <>
@@ -936,6 +937,10 @@ export default memo(function Map({
     dimensions.width < DESKTOP_TOP_CARDS_ROW_MIN_MAP_WIDTH;
   const showLocationSearch = !hideLocationSearch && !hideDesktopTopCardsForWidth;
   const showLegend = !hideLegend && !hideDesktopTopCardsForWidth;
+  const getViewportCenter = useCallback(() => {
+    const center = mapRef.current?.getCenter();
+    return center ? { lat: center.lat, lng: center.lng } : null;
+  }, []);
   const desktopTopCardsLayoutClass = stackDesktopTopCards
     ? 'flex-col items-start'
     : 'items-start justify-between';
@@ -1098,6 +1103,7 @@ export default memo(function Map({
                   onLocationSearched={onLocationSearched}
                   onCurrentLocationFound={onCurrentLocationFound}
                   onMouseEnter={handleMouseLeave}
+                  getViewportCenter={getViewportCenter}
                   className={DESKTOP_TOP_CARD_CLASS}
                   inputClassName={DESKTOP_LOCATION_SEARCH_INPUT_CLASS}
                 />

frontend/src/components/map/MapPage.tsx

@@ -91,6 +91,7 @@ export default function MapPage({
   onNavigateTo,
   onExportStateChange,
   onDashboardParamsChange,
+  onDashboardReadyChange,
   screenshotMode,
   ogMode,
   isMobile = false,
@@ -642,6 +643,23 @@ export default function MapPage({
     onDashboardParamsChange?.(dashboardParams);
   }, [dashboardParams, onDashboardParamsChange]);
 
+  const dashboardReady =
+    !initialLoading &&
+    !mapData.loading &&
+    !mapData.licenseRequired &&
+    mapData.bounds != null &&
+    mapData.currentView != null;
+
+  useEffect(() => {
+    onDashboardReadyChange?.(dashboardReady);
+  }, [dashboardReady, onDashboardReadyChange]);
+
+  useEffect(() => {
+    return () => {
+      onDashboardReadyChange?.(false);
+    };
+  }, [onDashboardReadyChange]);
+
   useEffect(() => {
     if (mapData.licenseRequired) trackEvent('Upgrade Modal Shown');
   }, [mapData.licenseRequired]);
@@ -830,8 +848,8 @@ export default function MapPage({
         </button>
         <button
           onClick={() => onUpdateEdit?.(dashboardParams)}
-          disabled={savingSearch}
-          className="shrink-0 cursor-pointer px-2.5 py-1 rounded text-xs font-medium bg-teal-600 text-white hover:bg-teal-700 disabled:opacity-50 disabled:cursor-wait flex items-center gap-1.5"
+          disabled={savingSearch || !dashboardReady}
+          className="shrink-0 cursor-pointer px-2.5 py-1 rounded text-xs font-medium bg-teal-600 text-white hover:bg-teal-700 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1.5"
         >
           {savingSearch ? t('savedPage.updating') : t('common.update')}
         </button>

frontend/src/components/map/MobileDrawer.test.tsx

@@ -0,0 +1,107 @@
+import { cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import MobileDrawer from './MobileDrawer';
+
+vi.mock('react-i18next', () => ({
+  useTranslation: () => ({
+    t: (key: string) => key,
+  }),
+}));
+
+const originalSetPointerCapture = HTMLElement.prototype.setPointerCapture;
+
+function renderDrawer(onClose = vi.fn()) {
+  const view = render(
+    <MobileDrawer
+      onClose={onClose}
+      renderArea={() => <div>Area content</div>}
+      renderProperties={() => <div>Properties content</div>}
+      tab="area"
+      onTabChange={vi.fn()}
+    />
+  );
+  const handle = view.container.querySelector('[data-mobile-drawer-drag-handle]');
+  const root = view.container.querySelector('[data-tutorial="right-pane"]');
+  const panel = view.container.querySelector('[data-tutorial="right-pane"] > div:last-child');
+
+  if (!(handle instanceof HTMLElement)) throw new Error('Expected drawer drag handle');
+  if (!(root instanceof HTMLElement)) throw new Error('Expected drawer root');
+  if (!(panel instanceof HTMLElement)) throw new Error('Expected drawer panel');
+
+  return { ...view, handle, onClose, panel, root };
+}
+
+describe('MobileDrawer', () => {
+  beforeEach(() => {
+    HTMLElement.prototype.setPointerCapture = vi.fn();
+  });
+
+  afterEach(() => {
+    cleanup();
+    Object.defineProperty(HTMLElement.prototype, 'setPointerCapture', {
+      configurable: true,
+      value: originalSetPointerCapture,
+    });
+  });
+
+  it('lowers and stays open when swiped down from the handle', () => {
+    const { handle, onClose, panel } = renderDrawer();
+
+    fireEvent.pointerDown(handle, { pointerId: 1, clientY: 120 });
+    fireEvent.pointerMove(handle, { pointerId: 1, clientY: 230 });
+    fireEvent.pointerUp(handle, { pointerId: 1, clientY: 230 });
+
+    expect(onClose).not.toHaveBeenCalled();
+    expect(panel.style.transform).toBe('translateY(110px)');
+  });
+
+  it('can be raised again after being lowered', () => {
+    const { handle, onClose, panel } = renderDrawer();
+
+    fireEvent.pointerDown(handle, { pointerId: 1, clientY: 120 });
+    fireEvent.pointerMove(handle, { pointerId: 1, clientY: 230 });
+    fireEvent.pointerUp(handle, { pointerId: 1, clientY: 230 });
+
+    fireEvent.pointerDown(handle, { pointerId: 2, clientY: 230 });
+    fireEvent.pointerMove(handle, { pointerId: 2, clientY: 170 });
+    fireEvent.pointerUp(handle, { pointerId: 2, clientY: 170 });
+
+    expect(onClose).not.toHaveBeenCalled();
+    expect(panel.style.transform).toBe('translateY(50px)');
+  });
+
+  it('keeps the close control reachable when dragged down far', () => {
+    const { handle, panel } = renderDrawer();
+
+    Object.defineProperty(panel, 'offsetHeight', {
+      configurable: true,
+      value: 200,
+    });
+
+    fireEvent.pointerDown(handle, { pointerId: 1, clientY: 120 });
+    fireEvent.pointerMove(handle, { pointerId: 1, clientY: 420 });
+    fireEvent.pointerUp(handle, { pointerId: 1, clientY: 420 });
+
+    expect(panel.style.transform).toBe('translateY(96px)');
+  });
+
+  it('leaves the rest of the mobile map usable while the panel is open', () => {
+    const { panel, root } = renderDrawer();
+    const spacer = root.firstElementChild;
+
+    if (!(spacer instanceof HTMLElement)) throw new Error('Expected drawer spacer');
+
+    expect(root.className).toContain('pointer-events-none');
+    expect(panel.className).toContain('pointer-events-auto');
+    expect(spacer.className).not.toContain('bg-black');
+  });
+
+  it('closes from the close button', () => {
+    const { onClose } = renderDrawer();
+
+    fireEvent.click(screen.getByLabelText('mobileDrawer.closeDrawer'));
+
+    expect(onClose).toHaveBeenCalledTimes(1);
+  });
+});

frontend/src/components/map/MobileDrawer.tsx

@@ -103,7 +103,10 @@ export default function MobileDrawer({
   }, []);
 
   return (
-    <div data-tutorial="right-pane" className="pointer-events-none fixed inset-0 z-50 flex flex-col">
+    <div
+      data-tutorial="right-pane"
+      className="pointer-events-none fixed inset-0 z-50 flex flex-col"
+    >
       <div className="h-[10%] shrink-0" aria-hidden="true" />
 
       {/* Panel — bottom 90% */}

frontend/src/components/map/POIPane.tsx

@@ -186,7 +186,7 @@ export default function POIPane({
               </div>
               {!isCollapsed && (
                 <div className="px-3 py-2">
-                  <PillGroup>
+                  <PillGroup wrap>
                     {group.categories.map((category) => {
                       const logo = getPoiCategoryLogoUrl(category);
                       return (

frontend/src/components/map/map-page/DesktopMapPage.tsx

@@ -269,7 +269,10 @@ export function DesktopMapPage({
           </div>
         )}
         {poiPaneOpen && (
-          <div className="absolute bottom-28 right-4 z-10 flex h-[60vh] min-h-0 w-80 flex-col overflow-hidden rounded-lg border border-warm-200 bg-white shadow-xl dark:border-warm-700 dark:bg-warm-900">
+          <div
+            className="absolute bottom-28 right-4 z-10 flex min-h-0 w-80 max-w-[calc(100%_-_2rem)] flex-col overflow-hidden rounded-lg border border-warm-200 bg-white shadow-xl dark:border-warm-700 dark:bg-warm-900"
+            style={{ height: 'min(30rem, calc(100vh - 10rem))' }}
+          >
             {poiPane}
           </div>
         )}

frontend/src/components/map/map-page/MobileMapPage.tsx

@@ -132,6 +132,10 @@ export function MobileMapPage({
   upgradeModal,
   editingBar,
 }: MobileMapPageProps) {
+  const floatingPaneAvailableHeight = `max(12rem, calc(100dvh - ${Math.ceil(
+    bottomScreenInset
+  )}px - 7rem))`;
+
   return (
     <div className="flex-1 overflow-hidden relative">
       <LoadingOverlay show={initialLoading} />
@@ -219,7 +223,13 @@ export function MobileMapPage({
       )}
 
       {poiPaneOpen && (
-        <div className="absolute top-24 right-3 left-3 z-20 flex h-[45dvh] min-h-0 flex-col overflow-hidden rounded-lg border border-warm-200 bg-white shadow-xl dark:border-warm-700 dark:bg-warm-900">
+        <div
+          className="absolute top-24 right-3 left-3 z-20 flex min-h-0 flex-col overflow-hidden rounded-lg border border-warm-200 bg-white shadow-xl dark:border-warm-700 dark:bg-warm-900"
+          style={{
+            height: `min(22rem, ${floatingPaneAvailableHeight})`,
+            maxHeight: floatingPaneAvailableHeight,
+          }}
+        >
           {poiPane}
         </div>
       )}

frontend/src/components/map/map-page/types.ts

@@ -39,6 +39,7 @@ export interface MapPageProps {
   onNavigateTo: (page: Page, hash?: string, infoFeature?: string) => void;
   onExportStateChange?: (state: ExportState) => void;
   onDashboardParamsChange?: (params: string) => void;
+  onDashboardReadyChange?: (ready: boolean) => void;
   screenshotMode?: boolean;
   ogMode?: boolean;
   isMobile?: boolean;

frontend/src/components/map/map-page/useExportController.ts

@@ -192,9 +192,7 @@ export function useExportController({
           const detail = err instanceof Error && err.message.trim() ? ` ${err.message}` : '';
           showExportNotice({
             kind: 'error',
-            message: timedOut
-              ? t('header.exportTimedOut')
-              : `${t('header.exportFailed')}${detail}`,
+            message: timedOut ? t('header.exportTimedOut') : `${t('header.exportFailed')}${detail}`,
           });
         } finally {
           window.clearTimeout(timeoutId);

frontend/src/components/ui/AuthModal.tsx

@@ -1,4 +1,4 @@
-import { useState, useCallback, useEffect } from 'react';
+import { useState, useCallback, useEffect, useId } from 'react';
 import { useTranslation } from 'react-i18next';
 import { CloseIcon } from './icons/CloseIcon';
 import { GoogleIcon } from './icons/GoogleIcon';
@@ -36,6 +36,9 @@ export default function AuthModal({
   const [password, setPassword] = useState('');
   const [resetSent, setResetSent] = useState(false);
   const dialogRef = useModalA11y();
+  const fieldId = useId();
+  const emailInputId = `${fieldId}-email`;
+  const passwordInputId = `${fieldId}-password`;
 
   useEffect(() => {
     trackEvent('Auth Modal Open', { tab: initialTab });
@@ -194,14 +197,20 @@ export default function AuthModal({
           {/* Email form */}
           <form onSubmit={handleSubmit} className="space-y-4">
             <div>
-              <label className="block text-sm font-medium text-warm-700 dark:text-warm-300 mb-1">
+              <label
+                htmlFor={emailInputId}
+                className="block text-sm font-medium text-warm-700 dark:text-warm-300 mb-1"
+              >
                 {t('auth.email')}
               </label>
               <input
+                id={emailInputId}
+                name="email"
                 type="email"
                 value={email}
                 onChange={(e) => setEmail(e.target.value)}
                 required
+                autoComplete="email"
                 className="w-full px-3 py-2 text-sm rounded border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-navy-950 dark:text-white placeholder-warm-400 dark:placeholder-warm-500 outline-none focus:ring-2 ring-teal-400 dark:ring-teal-500"
                 placeholder={t('auth.emailPlaceholder')}
               />
@@ -209,15 +218,21 @@ export default function AuthModal({
 
             {view !== 'forgot' && (
               <div>
-                <label className="block text-sm font-medium text-warm-700 dark:text-warm-300 mb-1">
+                <label
+                  htmlFor={passwordInputId}
+                  className="block text-sm font-medium text-warm-700 dark:text-warm-300 mb-1"
+                >
                   {t('auth.password')}
                 </label>
                 <input
+                  id={passwordInputId}
+                  name="password"
                   type="password"
                   value={password}
                   onChange={(e) => setPassword(e.target.value)}
                   required
                   minLength={8}
+                  autoComplete={view === 'register' ? 'new-password' : 'current-password'}
                   className="w-full px-3 py-2 text-sm rounded border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-navy-950 dark:text-white placeholder-warm-400 dark:placeholder-warm-500 outline-none focus:ring-2 ring-teal-400 dark:ring-teal-500"
                   placeholder={
                     view === 'register'

frontend/src/components/ui/ExportMenu.tsx

@@ -78,11 +78,7 @@ export default function ExportMenu({ open, exporting, onClose, onExport }: Expor
       } else {
         inputRefs.current[idx + 1]?.focus();
       }
-    } else if (
-      e.key === 'Backspace' &&
-      postcodes[idx] === '' &&
-      postcodes.length > 1
-    ) {
+    } else if (e.key === 'Backspace' && postcodes[idx] === '' && postcodes.length > 1) {
       e.preventDefault();
       focusIndexRef.current = Math.max(0, idx - 1);
       removeAt(idx);
@@ -98,11 +94,7 @@ export default function ExportMenu({ open, exporting, onClose, onExport }: Expor
 
   return (
     <>
-      <div
-        className="fixed inset-0 bg-black/50 z-[90]"
-        onClick={onClose}
-        aria-hidden="true"
-      />
+      <div className="fixed inset-0 bg-black/50 z-[90]" onClick={onClose} aria-hidden="true" />
       <div
         role="dialog"
         aria-modal="true"

frontend/src/components/ui/FeatureLabel.tsx

@@ -68,9 +68,7 @@ export function FeatureLabel({
       {GroupIcon && <GroupIcon className={iconClass} />}
       {translatedDesc ? (
         <div className="min-w-0">
-          <div className={`flex ${wrap ? 'items-start' : 'items-center'} gap-1`}>
-            {nameContent}
-          </div>
+          <div className={`flex ${wrap ? 'items-start' : 'items-center'} gap-1`}>{nameContent}</div>
           <span className="text-xs text-warm-400 dark:text-warm-500 block">{translatedDesc}</span>
         </div>
       ) : (

frontend/src/components/ui/Header.tsx

@@ -78,6 +78,7 @@ export default function Header({
   onToggleTheme,
   exportState,
   dashboardParams,
+  dashboardActionsDisabled = false,
   onSaveSearch,
   savingSearch,
   editingSearch,
@@ -96,6 +97,7 @@ export default function Header({
   onToggleTheme: () => void;
   exportState: HeaderExportState | null;
   dashboardParams: string;
+  dashboardActionsDisabled?: boolean;
   onSaveSearch: (() => void) | null;
   savingSearch: boolean;
   editingSearch: EditingSearchState | null;
@@ -116,6 +118,7 @@ export default function Header({
     () => window.matchMedia(DASHBOARD_TABLET_SIDEBAR_QUERY).matches
   );
   const useSidebarNav = isMobile || (activePage === 'dashboard' && isDashboardTabletSidebarWidth);
+  const dashboardActionsBlocked = activePage === 'dashboard' && (!user || dashboardActionsDisabled);
 
   useEffect(() => {
     const mql = window.matchMedia(DASHBOARD_TABLET_SIDEBAR_QUERY);
@@ -139,6 +142,10 @@ export default function Header({
     if (!useSidebarNav) setMenuOpen(false);
   }, [useSidebarNav]);
 
+  useEffect(() => {
+    if (dashboardActionsBlocked) setExportMenuOpen(false);
+  }, [dashboardActionsBlocked]);
+
   const doCopy = useCallback((text: string) => {
     copyToClipboard(text, () => {
       setCopied(true);
@@ -147,6 +154,7 @@ export default function Header({
   }, []);
 
   const handleShare = useCallback(async () => {
+    if (dashboardActionsBlocked) return;
     const params =
       activePage === 'dashboard' ? dashboardParams : window.location.search.replace(/^\?/, '');
     if (!params) {
@@ -167,7 +175,7 @@ export default function Header({
     } finally {
       setSharing(false);
     }
-  }, [activePage, dashboardParams, doCopy, i18n.language]);
+  }, [activePage, dashboardActionsBlocked, dashboardParams, doCopy, i18n.language]);
 
   const navLink = (page: Page, e: React.MouseEvent, hash?: string) => {
     if (e.metaKey || e.ctrlKey || e.shiftKey || e.button !== 0) return;
@@ -206,8 +214,8 @@ export default function Header({
               </button>
               <button
                 onClick={onUpdateEdit}
-                disabled={savingSearch}
-                className="cursor-pointer px-3 py-1.5 rounded bg-teal-600 hover:bg-teal-700 transition-colors text-sm font-medium disabled:opacity-50 disabled:cursor-wait flex items-center gap-1.5"
+                disabled={savingSearch || dashboardActionsBlocked}
+                className="cursor-pointer px-3 py-1.5 rounded bg-teal-600 hover:bg-teal-700 transition-colors text-sm font-medium disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1.5"
               >
                 {savingSearch && <SpinnerIcon className="w-4 h-4 animate-spin" />}
                 {savingSearch ? t('savedPage.updating') : t('common.update')}
@@ -216,14 +224,16 @@ export default function Header({
           </div>
         )}
         {/* Left: Logo + nav */}
-        <div className="flex items-center gap-4">
+        <div className="flex min-w-0 items-center gap-4">
           <a
             href="/"
-            className="flex cursor-pointer items-center gap-2 hover:opacity-80 transition-opacity"
+            className="flex min-w-0 cursor-pointer items-center gap-2 hover:opacity-80 transition-opacity"
             onClick={(e) => navLink('home', e)}
           >
-            <LogoIcon className="w-5 h-5 text-teal-400" />
-            <span className="text-lg font-semibold text-teal-300">{t('header.appName')}</span>
+            <LogoIcon className="w-5 h-5 shrink-0 text-teal-400" />
+            <span className="max-w-[9rem] truncate whitespace-nowrap text-lg font-semibold text-teal-300 sm:max-w-none">
+              {t('header.appName')}
+            </span>
           </a>
 
           {/* Desktop nav */}
@@ -266,14 +276,14 @@ export default function Header({
         </div>
 
         {/* Right side */}
-        <div className="flex items-center gap-2 ml-auto">
+        <div className="ml-auto flex shrink-0 items-center gap-2">
           {/* Desktop-only dashboard actions */}
-          {!useSidebarNav && activePage === 'dashboard' && (
+          {!useSidebarNav && activePage === 'dashboard' && user && (
             <>
               <button
                 onClick={handleShare}
-                disabled={sharing}
-                className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:opacity-50"
+                disabled={sharing || dashboardActionsBlocked}
+                className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:cursor-not-allowed disabled:opacity-50"
               >
                 {sharing ? (
                   <>
@@ -295,8 +305,8 @@ export default function Header({
               {exportState && (
                 <button
                   onClick={() => setExportMenuOpen(true)}
-                  disabled={exportState.exporting}
-                  className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:opacity-50"
+                  disabled={exportState.exporting || dashboardActionsBlocked}
+                  className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:cursor-not-allowed disabled:opacity-50"
                   title={t('header.exportToExcel')}
                 >
                   <DownloadIcon className="w-4 h-4" />
@@ -306,8 +316,8 @@ export default function Header({
               {onSaveSearch && !editingSearch && (
                 <button
                   onClick={onSaveSearch}
-                  disabled={savingSearch}
-                  className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:opacity-50"
+                  disabled={savingSearch || dashboardActionsBlocked}
+                  className="flex cursor-pointer items-center gap-1.5 px-3 py-1.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm disabled:cursor-not-allowed disabled:opacity-50"
                 >
                   {savingSearch ? (
                     <SpinnerIcon className="w-4 h-4 animate-spin" />
@@ -363,7 +373,7 @@ export default function Header({
           {useSidebarNav && !user && (
             <button
               onClick={onRegisterClick}
-              className="cursor-pointer px-4 py-1.5 rounded bg-teal-600 hover:bg-teal-700 transition-colors text-sm font-semibold"
+              className="flex h-8 max-w-[8.5rem] shrink-0 cursor-pointer items-center justify-center truncate whitespace-nowrap rounded bg-teal-600 px-2.5 text-xs font-semibold leading-none transition-colors hover:bg-teal-700 sm:max-w-none sm:px-3 sm:text-sm"
             >
               {t('header.createAccount')}
             </button>
@@ -410,6 +420,7 @@ export default function Header({
           onToggleTheme={onToggleTheme}
           exportState={exportState}
           onOpenExportMenu={() => setExportMenuOpen(true)}
+          dashboardActionsDisabled={dashboardActionsBlocked}
           onSaveSearch={onSaveSearch}
           savingSearch={savingSearch}
           isEditingSearch={!!editingSearch}

frontend/src/components/ui/MobileMenu.tsx

@@ -20,6 +20,7 @@ interface MobileMenuProps {
   onToggleTheme: () => void;
   exportState: HeaderExportState | null;
   onOpenExportMenu: () => void;
+  dashboardActionsDisabled?: boolean;
   onSaveSearch: (() => void) | null;
   savingSearch: boolean;
   isEditingSearch: boolean;
@@ -41,6 +42,7 @@ export default function MobileMenu({
   onToggleTheme,
   exportState,
   onOpenExportMenu,
+  dashboardActionsDisabled = false,
   onSaveSearch,
   savingSearch,
   isEditingSearch,
@@ -101,7 +103,7 @@ export default function MobileMenu({
     </a>
   );
 
-  const dashboardActions = activePage === 'dashboard' && (
+  const dashboardActions = activePage === 'dashboard' && user && (
     <div className="px-2 py-2 border-b border-navy-700">
       <div className="grid grid-cols-2 gap-2">
         <button
@@ -109,7 +111,7 @@ export default function MobileMenu({
             onShare();
             onClose();
           }}
-          disabled={sharing}
+          disabled={sharing || dashboardActionsDisabled}
           className={dashboardActionClass}
         >
           {sharing ? (
@@ -127,8 +129,8 @@ export default function MobileMenu({
               onClose();
               onOpenExportMenu();
             }}
-            disabled={exportState.exporting}
             className={dashboardActionClass}
+            disabled={exportState.exporting || dashboardActionsDisabled}
           >
             <DownloadIcon className="w-4 h-4" />
             {exportState.exporting ? t('header.exporting') : t('header.exportLabel')}
@@ -140,7 +142,7 @@ export default function MobileMenu({
               onSaveSearch();
               onClose();
             }}
-            disabled={savingSearch}
+            disabled={savingSearch || dashboardActionsDisabled}
             className={dashboardActionClass}
           >
             {savingSearch ? (
@@ -199,7 +201,7 @@ export default function MobileMenu({
           </button>
 
           {/* Language selector */}
-          <div className="flex max-w-full gap-1 overflow-x-auto overflow-y-hidden px-3 pb-1 scrollbar-hide">
+          <div className="grid max-w-full grid-cols-3 gap-1 px-3 pb-1">
             {SUPPORTED_LANGUAGES.map((lang) => (
               <button
                 key={lang.code}
@@ -208,7 +210,7 @@ export default function MobileMenu({
                   localStorage.setItem('language', lang.code);
                   void changeAppLanguage(lang.code);
                 }}
-                className={`flex-none min-w-[2.5rem] flex cursor-pointer items-center justify-center gap-1.5 px-2 py-1.5 rounded text-sm ${
+                className={`flex min-w-0 cursor-pointer items-center justify-center gap-1.5 rounded px-2 py-1.5 text-sm ${
                   i18n.language === lang.code
                     ? 'bg-navy-700 text-white font-medium'
                     : 'text-warm-400 hover:bg-navy-800 hover:text-white'

frontend/src/components/ui/PillGroup.tsx

@@ -3,12 +3,17 @@ import type { ReactNode } from 'react';
 interface PillGroupProps {
   children: ReactNode;
   className?: string;
+  wrap?: boolean;
 }
 
-export function PillGroup({ children, className = '' }: PillGroupProps) {
+export function PillGroup({ children, className = '', wrap = false }: PillGroupProps) {
   return (
     <div
-      className={`flex min-w-0 max-w-full flex-nowrap gap-1.5 overflow-x-auto overscroll-x-contain touch-pan-x touch-pan-y scrollbar-hide md:flex-wrap md:overflow-x-visible ${className}`}
+      className={`flex min-w-0 max-w-full gap-1.5 ${
+        wrap
+          ? 'flex-wrap overflow-x-visible'
+          : 'flex-nowrap overflow-x-auto overscroll-x-contain touch-pan-x touch-pan-y scrollbar-hide md:flex-wrap md:overflow-x-visible'
+      } ${className}`}
     >
       {children}
     </div>

frontend/src/components/ui/PlaceSearchInput.tsx

@@ -1,5 +1,6 @@
 import { useRef } from 'react';
 import { createPortal } from 'react-dom';
+import { useTranslation } from 'react-i18next';
 import type React from 'react';
 import type { SearchResult } from '../../hooks/useLocationSearch';
 import { useDropdownPosition } from '../../hooks/useDropdownPosition';
@@ -13,6 +14,7 @@ interface SearchHook {
   activeIndex: number;
   setActiveIndex: (idx: number) => void;
   open: boolean;
+  searching?: boolean;
   handleInputChange: (value: string) => void;
   handleKeyDown: (e: React.KeyboardEvent, onSelect: (result: SearchResult) => void) => void;
   showEmptySearches: () => void;
@@ -23,6 +25,8 @@ interface PlaceSearchInputProps {
   onSelect: (result: SearchResult) => void;
   loading?: boolean;
   placeholder?: string;
+  ariaLabel?: string;
+  name?: string;
   size?: 'sm' | 'xs';
   inputClassName?: string;
   inputRef?: React.Ref<HTMLInputElement>;
@@ -35,19 +39,28 @@ export function PlaceSearchInput({
   onSelect,
   loading,
   placeholder,
+  ariaLabel,
+  name,
   size = 'sm',
   inputClassName,
   inputRef,
   onInputChange,
   portal,
 }: PlaceSearchInputProps) {
+  const { t } = useTranslation();
   const sm = size === 'sm';
   const iconSize = sm ? 'w-4 h-4' : 'w-3 h-3';
   const spinnerSize = sm ? 'w-4 h-4' : 'w-3 h-3';
   const wrapperRef = useRef<HTMLDivElement>(null);
   const dropdownPos = useDropdownPosition(wrapperRef, portal ? search.open : false);
 
-  const showDropdown = search.open && search.results.length > 0;
+  const showEmptyResults =
+    search.open &&
+    !search.searching &&
+    search.query.trim().length >= 2 &&
+    search.results.length === 0;
+  const showDropdown = search.open && (search.results.length > 0 || showEmptyResults);
+  const showSpinner = loading || search.searching;
 
   const dropdown = showDropdown && (
     <div
@@ -64,7 +77,15 @@ export function PlaceSearchInput({
           : undefined
       }
     >
-      {search.results.map((result, idx) => (
+      {showEmptyResults ? (
+        <div
+          className={`text-warm-500 dark:text-warm-400 ${sm ? 'px-3 py-2 text-sm' : 'px-2 py-1.5 text-xs'}`}
+          role="status"
+        >
+          {t('locationSearch.noResults')}
+        </div>
+      ) : (
+        search.results.map((result, idx) => (
           <button
             key={
               result.type === 'postcode'
@@ -114,7 +135,8 @@ export function PlaceSearchInput({
               </>
             )}
           </button>
-      ))}
+        ))
+      )}
     </div>
   );
 
@@ -123,6 +145,7 @@ export function PlaceSearchInput({
       <input
         ref={inputRef}
         type="text"
+        name={name}
         value={search.query}
         onChange={(e) => {
           search.handleInputChange(e.target.value);
@@ -132,11 +155,12 @@ export function PlaceSearchInput({
           search.showEmptySearches();
         }}
         onKeyDown={(e) => search.handleKeyDown(e, onSelect)}
+        aria-label={ariaLabel ?? placeholder}
         placeholder={placeholder}
         className={inputClassName}
       />
 
-      {loading && (
+      {showSpinner && (
         <div
           className={`absolute right-2 top-1/2 -translate-y-1/2 ${spinnerSize} border-2 border-warm-300 dark:border-warm-600 border-t-teal-500 rounded-full animate-spin`}
         />

frontend/src/components/ui/SearchInput.tsx

@@ -4,18 +4,27 @@ interface SearchInputProps {
   value: string;
   onChange: (value: string) => void;
   placeholder?: string;
+  ariaLabel?: string;
   className?: string;
 }
 
-export function SearchInput({ value, onChange, placeholder, className = '' }: SearchInputProps) {
+export function SearchInput({
+  value,
+  onChange,
+  placeholder,
+  ariaLabel,
+  className = '',
+}: SearchInputProps) {
   const { t } = useTranslation();
+  const inputPlaceholder = placeholder ?? t('common.search');
 
   return (
     <input
       type="text"
       value={value}
       onChange={(e) => onChange(e.target.value)}
-      placeholder={placeholder ?? t('common.search')}
+      placeholder={inputPlaceholder}
+      aria-label={ariaLabel ?? inputPlaceholder}
       className={`w-full px-2 py-1 text-sm border rounded bg-white dark:bg-navy-800 dark:text-warm-200 border-warm-200 dark:border-navy-700 placeholder-warm-400 dark:placeholder-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400 ${className}`}
     />
   );

frontend/src/hooks/useHexagonSelection.ts

@@ -144,12 +144,7 @@ export function useHexagonSelection({
   );
 
   const fetchPostcodeStats = useCallback(
-    async (
-      postcode: string,
-      signal?: AbortSignal,
-      includeFilters = true,
-      fields?: string[]
-    ) => {
+    async (postcode: string, signal?: AbortSignal, includeFilters = true, fields?: string[]) => {
       const params = new URLSearchParams({ postcode });
       const filterStr = includeFilters ? buildFilterString(filters, features) : '';
       if (filterStr) params.append('filters', filterStr);

frontend/src/hooks/useListingLayers.ts

@@ -13,6 +13,7 @@ const LISTING_CLUSTER_MAX_ZOOM = 24;
 const LISTING_CLUSTER_POPUP_LIMIT = 30;
 const LISTING_SPIDERFY_LIMIT = 12;
 const TILE_SIZE = 512;
+const PRICE_LABEL_CHARACTER_SET = '£0123456789.kM';
 
 interface SingleListingPopupInfo {
   mode: 'single';
@@ -472,6 +473,7 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
       outlineWidth: 3,
       outlineColor: isDark ? [10, 10, 10, 220] : [255, 255, 255, 230],
       fontSettings: { sdf: true },
+      characterSet: PRICE_LABEL_CHARACTER_SET,
       sizeUnits: 'pixels',
       sizeMinPixels: 10,
       sizeMaxPixels: 14,

frontend/src/hooks/useLocationSearch.ts

@@ -154,69 +154,63 @@ function searchResultKey(result: SearchResult): string {
   return `place:${result.slug}`;
 }
 
-export function useLocationSearch(mode?: string) {
-  const [query, setQuery] = useState('');
-  const [results, setResults] = useState<SearchResult[]>([]);
-  const [recentSearches, setRecentSearches] = useState<SearchResult[]>(readRecentSearches);
-  const [activeIndex, setActiveIndex] = useState(-1);
-  const [open, setOpen] = useState(false);
-  const abortRef = useRef<AbortController | null>(null);
-  const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
-  const latestQueryRef = useRef('');
-  const lastResultsRef = useRef<SearchResult[]>([]);
+/** A category-tagged, scored result from the unified `/api/places` ranking. */
+type UnifiedResultDTO =
+  | { type: 'postcode'; label: string; score: number }
+  | { type: 'address'; address: string; postcode: string; lat: number; lon: number; score: number }
+  | {
+      type: 'place';
+      name: string;
+      slug: string;
+      place_type: string;
+      lat: number;
+      lon: number;
+      city?: string;
+      score: number;
+    };
 
-  const handleInputChange = useCallback(
-    (value: string) => {
-      setQuery(value);
-      latestQueryRef.current = value;
-      setActiveIndex(-1);
-
-      abortRef.current?.abort();
-      if (debounceRef.current) clearTimeout(debounceRef.current);
-
-      const trimmed = value.trim();
-      if (!trimmed) {
-        setResults(recentSearches);
-        lastResultsRef.current = [];
-        setOpen(recentSearches.length > 0);
-        return;
-      }
-
-      if (!mode && looksLikePostcode(trimmed)) {
-        const postcodeResults: SearchResult[] = [
-          { type: 'postcode', label: normalizePostcode(trimmed) },
-        ];
-        setResults(postcodeResults);
-        setOpen(true);
-        return;
-      }
-
-      if (trimmed.length < 2) {
-        setResults([]);
-        setOpen(false);
-        return;
-      }
-
-      const locallyFilteredResults = filterResultsForQuery(lastResultsRef.current, trimmed);
-      setResults(locallyFilteredResults);
-      setOpen(locallyFilteredResults.length > 0);
-
-      debounceRef.current = setTimeout(async () => {
-        const controller = new AbortController();
-        abortRef.current = controller;
-        try {
-          const params = new URLSearchParams({ q: trimmed });
-          if (mode) params.set('mode', mode);
-          const res = await fetch(
-            `/api/places?${params}`,
-            authHeaders({ signal: controller.signal })
-          );
-          if (!res.ok) return;
-          const json: {
+interface PlacesApiResponse {
   places: PlaceResult[];
   postcodes?: string[];
   addresses?: AddressResult[];
-          } = await res.json();
+  /** Preferred: a single relevance-ordered list across all categories. */
+  results?: UnifiedResultDTO[];
+}
+
+function isNonNull<T>(value: T | null): value is T {
+  return value !== null;
+}
+
+function unifiedToSearchResult(result: UnifiedResultDTO): SearchResult | null {
+  if (result.type === 'postcode') {
+    return { type: 'postcode', label: result.label };
+  }
+  if (result.type === 'address') {
+    return {
+      type: 'address',
+      address: result.address,
+      postcode: result.postcode,
+      lat: result.lat,
+      lon: result.lon,
+    };
+  }
+  if (result.type === 'place') {
+    return {
+      type: 'place',
+      name: result.name,
+      slug: result.slug,
+      place_type: result.place_type,
+      lat: result.lat,
+      lon: result.lon,
+      city: result.city === 'City of London' ? 'London' : result.city,
+    };
+  }
+  return null;
+}
+
+/** Legacy ordering for servers that predate the unified `results` list: positional buckets,
+ *  re-filtered locally. Retained only as a fallback. */
+function legacyCombineResults(json: PlacesApiResponse, trimmed: string): SearchResult[] {
   const placeResults = json.places.map((p) => ({
     type: 'place' as const,
     name: p.name,
@@ -227,9 +221,7 @@ export function useLocationSearch(mode?: string) {
     city: p.city === 'City of London' ? 'London' : p.city,
   }));
   const outcodeResults = placeResults.filter((result) => result.place_type === 'outcode');
-          const otherPlaceResults = placeResults.filter(
-            (result) => result.place_type !== 'outcode'
-          );
+  const otherPlaceResults = placeResults.filter((result) => result.place_type !== 'outcode');
   const postcodeResults: SearchResult[] = (json.postcodes ?? []).map((postcode) => ({
     type: 'postcode' as const,
     label: postcode,
@@ -242,20 +234,117 @@ export function useLocationSearch(mode?: string) {
     lon: address.lon,
   }));
   const containsHouseNumber = /\d/.test(trimmed);
-          const combinedResults = (
+  return filterResultsForQuery(
     containsHouseNumber
       ? [...outcodeResults, ...postcodeResults, ...addressResults, ...otherPlaceResults]
-              : [...outcodeResults, ...postcodeResults, ...otherPlaceResults, ...addressResults]
+      : [...outcodeResults, ...postcodeResults, ...otherPlaceResults, ...addressResults],
+    trimmed
+  );
+}
+
+export type ViewportCenter = { lat: number; lng: number };
+
+export function useLocationSearch(mode?: string, getViewportCenter?: () => ViewportCenter | null) {
+  const [query, setQuery] = useState('');
+  const [results, setResults] = useState<SearchResult[]>([]);
+  const [recentSearches, setRecentSearches] = useState<SearchResult[]>(readRecentSearches);
+  const [activeIndex, setActiveIndex] = useState(-1);
+  const [open, setOpen] = useState(false);
+  const [searching, setSearching] = useState(false);
+  const abortRef = useRef<AbortController | null>(null);
+  const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const latestQueryRef = useRef('');
+  const lastResultsRef = useRef<SearchResult[]>([]);
+  // Held in a ref so a non-memoized callback from the parent doesn't churn handleInputChange.
+  const getViewportCenterRef = useRef(getViewportCenter);
+  getViewportCenterRef.current = getViewportCenter;
+
+  const handleInputChange = useCallback(
+    (value: string) => {
+      setQuery(value);
+      latestQueryRef.current = value;
+      setActiveIndex(-1);
+
+      abortRef.current?.abort();
+      if (debounceRef.current) clearTimeout(debounceRef.current);
+
+      const trimmed = value.trim();
+      if (!trimmed) {
+        setSearching(false);
+        setResults(recentSearches);
+        lastResultsRef.current = [];
+        setOpen(recentSearches.length > 0);
+        return;
+      }
+
+      if (!mode && looksLikePostcode(trimmed)) {
+        setSearching(false);
+        const postcodeResults: SearchResult[] = [
+          { type: 'postcode', label: normalizePostcode(trimmed) },
+        ];
+        setResults(postcodeResults);
+        setOpen(true);
+        return;
+      }
+
+      if (trimmed.length < 2) {
+        setSearching(false);
+        setResults([]);
+        setOpen(false);
+        return;
+      }
+
+      const locallyFilteredResults = filterResultsForQuery(lastResultsRef.current, trimmed);
+      setResults(locallyFilteredResults);
+      setOpen(locallyFilteredResults.length > 0);
+      setSearching(true);
+
+      debounceRef.current = setTimeout(async () => {
+        const controller = new AbortController();
+        abortRef.current = controller;
+        try {
+          const params = new URLSearchParams({ q: trimmed });
+          if (mode) params.set('mode', mode);
+          const center = getViewportCenterRef.current?.();
+          if (center) {
+            params.set('lat', String(center.lat));
+            params.set('lng', String(center.lng));
+          }
+          const res = await fetch(
+            `/api/places?${params}`,
+            authHeaders({ signal: controller.signal })
+          );
+          if (!res.ok) {
+            if (!controller.signal.aborted && latestQueryRef.current.trim() === trimmed) {
+              setResults([]);
+              setOpen(true);
+            }
+            return;
+          }
+          const json: PlacesApiResponse = await res.json();
+          const combinedResults = (
+            Array.isArray(json.results)
+              ? json.results.map(unifiedToSearchResult).filter(isNonNull)
+              : legacyCombineResults(json, trimmed)
           ).slice(0, 20);
           if (controller.signal.aborted || latestQueryRef.current.trim() !== trimmed) {
             return;
           }
           lastResultsRef.current = combinedResults;
-          const matchingResults = filterResultsForQuery(combinedResults, trimmed);
-          setResults(matchingResults);
-          setOpen(matchingResults.length > 0);
+          // Trust the server's unified ranking — re-filtering here previously dropped valid
+          // alias and partial-postcode matches. The optimistic pre-fetch path still filters.
+          setResults(combinedResults);
+          setOpen(true);
         } catch (err) {
           logNonAbortError('places search', err);
+          if (!controller.signal.aborted && latestQueryRef.current.trim() === trimmed) {
+            setResults([]);
+            setOpen(true);
+          }
+        } finally {
+          if (!controller.signal.aborted && latestQueryRef.current.trim() === trimmed) {
+            setSearching(false);
+          }
         }
       }, 200);
     },
@@ -264,7 +353,7 @@ export function useLocationSearch(mode?: string) {
 
   const showEmptySearches = useCallback(() => {
     if (latestQueryRef.current.trim()) {
-      setOpen(results.length > 0);
+      setOpen(results.length > 0 || latestQueryRef.current.trim().length >= 2);
       return;
     }
 
@@ -278,6 +367,7 @@ export function useLocationSearch(mode?: string) {
   const clear = useCallback(() => {
     setQuery('');
     latestQueryRef.current = '';
+    setSearching(false);
     setResults([]);
     lastResultsRef.current = [];
     setOpen(false);
@@ -308,6 +398,8 @@ export function useLocationSearch(mode?: string) {
         e.preventDefault();
         if (activeIndex >= 0 && activeIndex < results.length) {
           onSelect(results[activeIndex]);
+        } else if (results.length > 0) {
+          onSelect(results[0]);
         } else if (looksLikePostcode(query)) {
           onSelect({ type: 'postcode', label: normalizePostcode(query) });
         }
@@ -332,6 +424,7 @@ export function useLocationSearch(mode?: string) {
     activeIndex,
     setActiveIndex,
     open,
+    searching,
     setOpen,
     handleInputChange,
     handleKeyDown,

frontend/src/hooks/useTravelTime.test.ts

@@ -178,7 +178,11 @@ describe('resolveTransitVariant', () => {
 
 describe('parseServerMode', () => {
   it('round-trips the four toggle-reachable variants', () => {
-    expect(parseServerMode('transit')).toEqual({ mode: 'transit', noChange: false, noBuses: false });
+    expect(parseServerMode('transit')).toEqual({
+      mode: 'transit',
+      noChange: false,
+      noBuses: false,
+    });
     expect(parseServerMode('transit-no-bus')).toEqual({
       mode: 'transit',
       noChange: false,
@@ -198,8 +202,16 @@ describe('parseServerMode', () => {
 
   it('parses non-transit base modes', () => {
     expect(parseServerMode('car')).toEqual({ mode: 'car', noChange: false, noBuses: false });
-    expect(parseServerMode('bicycle')).toEqual({ mode: 'bicycle', noChange: false, noBuses: false });
-    expect(parseServerMode('walking')).toEqual({ mode: 'walking', noChange: false, noBuses: false });
+    expect(parseServerMode('bicycle')).toEqual({
+      mode: 'bicycle',
+      noChange: false,
+      noBuses: false,
+    });
+    expect(parseServerMode('walking')).toEqual({
+      mode: 'walking',
+      noChange: false,
+      noBuses: false,
+    });
   });
 
   it('returns null for variants the UI cannot represent (no silent broadening)', () => {

frontend/src/hooks/useTravelTime.ts

@@ -138,7 +138,15 @@ export function useTravelTime(initial?: TravelTimeInitial) {
   const handleAddEntry = useCallback((mode: TransportMode) => {
     setEntries((prev) => [
       ...prev,
-      { mode, slug: '', label: '', timeRange: null, useBest: false, noChange: false, noBuses: false },
+      {
+        mode,
+        slug: '',
+        label: '',
+        timeRange: null,
+        useBest: false,
+        noChange: false,
+        noBuses: false,
+      },
     ]);
   }, []);
 

frontend/src/i18n/descriptions.ts

@@ -22,7 +22,8 @@ const descriptions: Record<string, Record<string, string>> = {
     'Est. price per sqm': 'Prix actuel estimé rapporté à la surface totale',
     'Estimated monthly rent': 'Loyer mensuel privé moyen dans le secteur',
     'Total floor area (sqm)': 'Surface intérieure relevée lors du diagnostic EPC',
-    'Number of bedrooms & living rooms': 'Nombre de pièces habitables relevé lors du diagnostic EPC',
+    'Number of bedrooms & living rooms':
+      'Nombre de pièces habitables relevé lors du diagnostic EPC',
     'Construction year': 'Année de construction estimée à partir de l’EPC',
     'Date of last transaction': 'Date de la vente la plus récente enregistrée au Land Registry',
     'Former council house': 'Indique si le bien a déjà été répertorié comme logement social',
@@ -30,7 +31,8 @@ const descriptions: Record<string, Record<string, string>> = {
     'Potential energy rating':
       'Classe énergétique EPC possible si toutes les améliorations recommandées étaient réalisées',
     'Interior height (m)': 'Hauteur intérieure moyenne relevée lors du diagnostic EPC',
-    'Street tree density percentile': 'Percentile estimé de couverture arborée autour du code postal',
+    'Street tree density percentile':
+      'Percentile estimé de couverture arborée autour du code postal',
     'Within conservation area':
       'Indique si le point représentatif du code postal se situe dans une zone de conservation désignée',
     'Listed building':
@@ -67,7 +69,8 @@ const descriptions: Record<string, Record<string, string>> = {
       'Moyenne annuelle des violences et infractions sexuelles dans le secteur',
     'Burglary (avg/yr)': 'Moyenne annuelle des cambriolages dans le secteur',
     'Robbery (avg/yr)': 'Moyenne annuelle des vols avec violence dans le secteur',
-    'Vehicle crime (avg/yr)': 'Moyenne annuelle des infractions liées aux véhicules dans le secteur',
+    'Vehicle crime (avg/yr)':
+      'Moyenne annuelle des infractions liées aux véhicules dans le secteur',
     'Anti-social behaviour (avg/yr)':
       'Moyenne annuelle des comportements antisociaux dans le secteur',
     'Criminal damage and arson (avg/yr)':
@@ -85,12 +88,11 @@ const descriptions: Record<string, Record<string, string>> = {
     '% White': 'Part de la population s’identifiant comme blanche',
     '% South Asian': 'Part de la population s’identifiant comme sud-asiatique',
     '% Black': 'Part de la population s’identifiant comme noire',
-    '% East Asian': 'Part de la population s’identifiant comme est-asiatique',
+    '% East/SE Asian': 'Part de la population s’identifiant comme est/sud-est asiatique',
     '% Mixed':
       'Part de la population s’identifiant comme métisse ou de plusieurs groupes ethniques',
     '% Other': 'Part de la population s’identifiant comme appartenant à un autre groupe ethnique',
-    'Voter turnout (%)':
-      'Part des électeurs inscrits ayant voté aux élections générales de 2024',
+    'Voter turnout (%)': 'Part des électeurs inscrits ayant voté aux élections générales de 2024',
     '% Labour': 'Part des voix travaillistes aux élections générales de 2024',
     '% Conservative': 'Part des voix conservatrices aux élections générales de 2024',
     '% Liberal Democrat': 'Part des voix libérales-démocrates aux élections générales de 2024',
@@ -98,8 +100,10 @@ const descriptions: Record<string, Record<string, string>> = {
     '% Green': 'Part des voix vertes aux élections générales de 2024',
     '% Other parties': 'Part cumulée des voix de tous les autres partis et indépendants',
     'Distance to nearest park (km)': 'Distance au parc ou espace vert le plus proche',
-    'Noise (dB)': 'Niveau maximal de bruit des transports près du code postal, en décibels (Lden)',
-    'Max available download speed (Mbps)': 'Débit descendant haut débit maximal disponible au code postal',
+    'Noise (dB)':
+      'Le plus élevé des bruits routier, ferroviaire ou aérien près du code postal, en décibels (Lden). Angleterre uniquement ; vide = hors zone cartographiée, pas forcément calme.',
+    'Max available download speed (Mbps)':
+      'Débit descendant haut débit maximal disponible au code postal',
     Schools: 'Écoles primaires et secondaires notées à proximité',
     'Specific crimes': 'Filtrer une seule catégorie d’infractions de rue à la fois',
     Ethnicities: 'Part de la population par groupe ethnique',
@@ -152,8 +156,7 @@ const descriptions: Record<string, Record<string, string>> = {
     'Education, Skills and Training Score':
       'Benachteiligungsperzentil für Bildung, Kompetenzen und Ausbildung (höher = weniger benachteiligt)',
     'Income Score': 'Einkommensbenachteiligungsperzentil (höher = weniger benachteiligt)',
-    'Employment Score':
-      'Beschäftigungsbenachteiligungsperzentil (höher = weniger benachteiligt)',
+    'Employment Score': 'Beschäftigungsbenachteiligungsperzentil (höher = weniger benachteiligt)',
     'Health Deprivation and Disability Score':
       'Perzentil für gesundheitliche Benachteiligung und Behinderung (höher = bessere Ergebnisse)',
     'Housing Conditions Score': 'Wohnbedingungen-Perzentil (höher = bessere Bedingungen)',
@@ -185,7 +188,7 @@ const descriptions: Record<string, Record<string, string>> = {
     '% White': 'Anteil der Personen, die sich als weiß identifizieren',
     '% South Asian': 'Anteil der Personen, die sich als südasiatisch identifizieren',
     '% Black': 'Anteil der Personen, die sich als schwarz identifizieren',
-    '% East Asian': 'Anteil der Personen, die sich als ostasiatisch identifizieren',
+    '% East/SE Asian': 'Anteil der Personen, die sich als ost-/südostasiatisch identifizieren',
     '% Mixed':
       'Anteil der Personen, die sich als gemischt oder mehreren ethnischen Gruppen zugehörig identifizieren',
     '% Other': 'Anteil der Personen, die sich einer anderen ethnischen Gruppe zuordnen',
@@ -198,7 +201,8 @@ const descriptions: Record<string, Record<string, string>> = {
     '% Green': 'Stimmenanteil der Grünen bei der Parlamentswahl 2024',
     '% Other parties': 'Kombinierter Stimmenanteil aller anderen Parteien und Unabhängigen',
     'Distance to nearest park (km)': 'Entfernung zum nächsten Park oder Grünfläche',
-    'Noise (dB)': 'Maximaler Verkehrslärmpegel in der Nähe des Postcodes in Dezibel (Lden)',
+    'Noise (dB)':
+      'Lautester von Straßen-, Bahn- oder Fluglärm in der Nähe des Postcodes in Dezibel (Lden). Nur England; leer = nicht kartiert, nicht unbedingt leise.',
     'Max available download speed (Mbps)':
       'Maximal verfügbare Breitband-Downloadgeschwindigkeit am Postcode',
     Schools: 'Bewertete Grundschulen und weiterführende Schulen in der Nähe',
@@ -262,7 +266,7 @@ const descriptions: Record<string, Record<string, string>> = {
     '% White': '白人人口比例',
     '% South Asian': '南亚裔人口比例',
     '% Black': '黑人人口比例',
-    '% East Asian': '东亚裔人口比例',
+    '% East/SE Asian': '东亚/东南亚裔人口比例',
     '% Mixed': '混血或多族裔人口比例',
     '% Other': '其他族裔人口比例',
     'Voter turnout (%)': '2024 年大选中登记选民的投票率',
@@ -273,7 +277,8 @@ const descriptions: Record<string, Record<string, string>> = {
     '% Green': '2024 年大选中绿党得票率',
     '% Other parties': '所有其他政党和独立候选人的综合得票率',
     'Distance to nearest park (km)': '到最近公园或绿地的距离',
-    'Noise (dB)': '该邮编附近最高交通噪音水平（Lden，分贝）',
+    'Noise (dB)':
+      '该邮编附近道路、铁路或机场中最高的噪音水平（Lden，分贝）。仅英格兰；空白表示未覆盖，不一定安静。',
     'Max available download speed (Mbps)': '该邮编可用的最高宽带下载速度',
     Schools: '附近有评级的小学和中学',
     'Specific crimes': '一次筛选一种街面犯罪类别',
@@ -347,7 +352,7 @@ const descriptions: Record<string, Record<string, string>> = {
     '% White': 'श्वेत के रूप में पहचान करने वाली आबादी का प्रतिशत',
     '% South Asian': 'दक्षिण एशियाई के रूप में पहचान करने वाली आबादी का प्रतिशत',
     '% Black': 'अश्वेत के रूप में पहचान करने वाली आबादी का प्रतिशत',
-    '% East Asian': 'पूर्वी एशियाई के रूप में पहचान करने वाली आबादी का प्रतिशत',
+    '% East/SE Asian': 'पूर्वी/दक्षिण-पूर्वी एशियाई के रूप में पहचान करने वाली आबादी का प्रतिशत',
     '% Mixed': 'मिश्रित या कई जातीय समूहों से पहचान करने वाली आबादी का प्रतिशत',
     '% Other': 'अन्य जातीय समूह के रूप में पहचान करने वाली आबादी का प्रतिशत',
     'Voter turnout (%)': '2024 आम चुनाव में मतदान करने वाले पंजीकृत मतदाताओं का प्रतिशत',
@@ -358,7 +363,8 @@ const descriptions: Record<string, Record<string, string>> = {
     '% Green': '2024 आम चुनाव में ग्रीन पार्टी का मत-प्रतिशत',
     '% Other parties': 'बाकी सभी पार्टियों और निर्दलीयों का संयुक्त मत-प्रतिशत',
     'Distance to nearest park (km)': 'निकटतम पार्क या हरित क्षेत्र तक दूरी',
-    'Noise (dB)': 'पोस्टकोड पर सड़क शोर स्तर, डेसीबल (Lden) में',
+    'Noise (dB)':
+      'पोस्टकोड के पास सड़क, रेल या हवाई अड्डे के शोर में सबसे अधिक, डेसीबल (Lden) में। केवल इंग्लैंड; खाली = मैप नहीं किया गया, जरूरी नहीं कि शांत हो।',
     'Max available download speed (Mbps)': 'पोस्टकोड पर उपलब्ध अधिकतम डाउनलोड गति',
     Schools: 'पास के रेटेड प्राइमरी और सेकेंडरी स्कूल',
     'Specific crimes': 'एक समय में एक सड़क-स्तर अपराध श्रेणी से फिल्टर करें',
@@ -438,7 +444,7 @@ const descriptions: Record<string, Record<string, string>> = {
     '% White': 'A fehérként azonosított lakosság aránya',
     '% South Asian': 'A dél-ázsiaiként azonosított lakosság aránya',
     '% Black': 'A feketeként azonosított lakosság aránya',
-    '% East Asian': 'A kelet-ázsiaiként azonosított lakosság aránya',
+    '% East/SE Asian': 'A kelet-/délkelet-ázsiaiként azonosított lakosság aránya',
     '% Mixed': 'A vegyes vagy több etnikai csoporthoz tartozóként azonosított lakosság aránya',
     '% Other': 'Az egyéb etnikai csoportba tartozóként azonosított lakosság aránya',
     'Voter turnout (%)':
@@ -450,7 +456,8 @@ const descriptions: Record<string, Record<string, string>> = {
     '% Green': 'A Zöld Párt szavazataránya a 2024-es parlamenti választáson',
     '% Other parties': 'Az összes többi párt és független jelölt összesített szavazataránya',
     'Distance to nearest park (km)': 'Távolság a legközelebbi parkig vagy zöldterületig',
-    'Noise (dB)': 'Közúti zajszint az irányítószámnál decibelben (Lden)',
+    'Noise (dB)':
+      'Az út-, vasúti vagy repülőtéri zaj közül a leghangosabb az irányítószámnál, decibelben (Lden). Csak Anglia; üres = nem térképezett, nem feltétlenül csendes.',
     'Max available download speed (Mbps)':
       'Az irányítószámnál elérhető maximális szélessávú letöltési sebesség',
     Schools: 'Közeli minősített általános és középiskolák',

frontend/src/i18n/details.ts

@@ -8,7 +8,7 @@ export const details: Record<string, Record<string, string>> = {
     'Property type':
       'Données HM Land Registry Price Paid et certificats EPC. Maison individuelle, maison jumelée, maison mitoyenne (tous les sous-types de maisons en rangée), appartement/maisonette, ou autre type (bungalows, park homes, etc.).',
     'Leasehold/Freehold':
-      "Données HM Land Registry Price Paid. Freehold signifie que vous possédez le bâtiment et le terrain sur lequel il se trouve. Leasehold signifie que vous possédez le bâtiment mais pas le terrain : vous détenez un bail accordé par le freeholder pour une durée déterminée.",
+      'Données HM Land Registry Price Paid. Freehold signifie que vous possédez le bâtiment et le terrain sur lequel il se trouve. Leasehold signifie que vous possédez le bâtiment mais pas le terrain : vous détenez un bail accordé par le freeholder pour une durée déterminée.',
     'Last known price':
       "Le dernier prix de vente enregistré pour ce bien provenant des données HM Land Registry Price Paid. Couvre les ventes résidentielles en Angleterre. Peut dater de plusieurs années si le bien n'a pas été vendu récemment.",
     'Estimated current price':
@@ -72,7 +72,7 @@ export const details: Record<string, Record<string, string>> = {
     'Serious crime (avg/yr)':
       "Somme annuelle des violences, robberies, burglaries et possessions d'armes dans un rayon de 50 m du code postal, comptée à partir des points de criminalité street-level de police.uk (anonymisés et rattachés à des points cartographiques proches). Fournit un indicateur unique de criminalité grave.",
     'Minor crime (avg/yr)':
-      "Somme annuelle des comportements antisociaux, shoplifting, vols de vélos et autres infractions de moindre gravité dans un rayon de 50 m du code postal, comptée à partir des points de criminalité street-level de police.uk (anonymisés et rattachés à des points cartographiques proches). Fournit un indicateur unique de criminalité mineure.",
+      'Somme annuelle des comportements antisociaux, shoplifting, vols de vélos et autres infractions de moindre gravité dans un rayon de 50 m du code postal, comptée à partir des points de criminalité street-level de police.uk (anonymisés et rattachés à des points cartographiques proches). Fournit un indicateur unique de criminalité mineure.',
     'Violence and sexual offences (avg/yr)':
       'Nombre moyen annuel de violences et infractions sexuelles dans un rayon de 50 m du code postal, d’après les données de criminalité street-level de police.uk. Inclut les agressions, le harcèlement et les infractions sexuelles.',
     'Burglary (avg/yr)':
@@ -109,8 +109,8 @@ export const details: Record<string, Record<string, string>> = {
       "Provient du Census 2021. Pourcentage de la population de l'autorité locale s'identifiant comme Indien, Pakistanais, Bangladais ou toute autre origine asiatique.",
     '% Black':
       "Provient du Census 2021. Pourcentage de la population de l'autorité locale s'identifiant comme Noir, Noir britannique, Caribéen ou Africain.",
-    '% East Asian':
-      "Provient du Census 2021. Pourcentage de la population de l'autorité locale s'identifiant comme Chinois.",
+    '% East/SE Asian':
+      "Provient du Census 2021. Pourcentage de la population de l'autorité locale s'identifiant comme Chinois ou d'une autre origine est/sud-est asiatique.",
     '% Mixed':
       "Provient du Census 2021. Pourcentage de la population de l'autorité locale s'identifiant comme Mixte ou appartenant à plusieurs groupes ethniques (Blanc et Noir caribéen, Blanc et Noir africain, Blanc et Asiatique, ou tout autre fond mixte ou multiple).",
     '% Other':
@@ -132,7 +132,7 @@ export const details: Record<string, Record<string, string>> = {
     'Distance to nearest park (km)':
       "Distance à vol d'oiseau en kilomètres entre le code postal et l'entrée de parc la plus proche. Couvre les parcs publics, jardins, terrains de sport et aires de jeux. Utilise les points d'accès du jeu de données OS Open Greenspace, afin que les biens en bordure d'un grand parc affichent bien une courte distance.",
     'Noise (dB)':
-      "Niveau maximal de bruit routier, ferroviaire ou aérien en décibels (Lden, moyenne pondérée sur 24 heures) d'après le Strategic Noise Mapping Round 4 de Defra (2022). Modélisé à 4 m au-dessus du sol sur une grille de 10 m et échantillonné comme la cellule de 10 m la plus bruyante autour du point représentatif du code postal. Au-dessus d'environ 55 dB, le bruit est généralement perceptible ; au-dessus d'environ 70 dB, l'OMS le considère comme nocif.",
+      "Niveau maximal de bruit routier, ferroviaire ou aérien en décibels (Lden, moyenne pondérée sur 24 heures) d'après le Strategic Noise Mapping Round 4 de Defra (2022). Modélisé à 4 m au-dessus du sol sur une grille de 10 m et échantillonné comme la cellule de 10 m la plus bruyante autour du point représentatif du code postal. Au-dessus d'environ 55 dB, le bruit est généralement perceptible ; au-dessus d'environ 70 dB, l'OMS le considère comme nocif. Couvre l'Angleterre uniquement ; une valeur vide signifie l'absence de données cartographiées (pas forcément calme).",
     'Max available download speed (Mbps)':
       "Débit descendant maximal de haut débit fixe disponible auprès de n'importe quel fournisseur, d'après Ofcom Connected Nations 2025. Il s'agit du maximum théorique, pas des débits réellement obtenus. 10 Mbps = basique, 30 = superfast, 100+ = ultrafast, 1000 = gigabit.",
     Schools:
@@ -255,8 +255,8 @@ export const details: Record<string, Record<string, string>> = {
       'Aus dem Census 2021. Prozentsatz der Bevölkerung der Gemeinde, die sich als Indisch, Pakistanisch, Bangladeschisch oder mit sonstigem asiatischen Hintergrund identifiziert.',
     '% Black':
       'Aus dem Census 2021. Prozentsatz der Bevölkerung der Gemeinde, die sich als Schwarz, Schwarz-Britisch, Karibisch oder Afrikanisch identifiziert.',
-    '% East Asian':
-      'Aus dem Census 2021. Prozentsatz der Bevölkerung der Gemeinde, die sich als Chinesisch identifiziert.',
+    '% East/SE Asian':
+      'Aus dem Census 2021. Prozentsatz der Bevölkerung der Gemeinde, die sich als Chinesisch oder einer anderen ost-/südostasiatischen Herkunft identifiziert.',
     '% Mixed':
       'Aus dem Census 2021. Prozentsatz der Bevölkerung der Gemeinde, die sich als gemischt oder mit mehreren ethnischen Zugehörigkeiten identifiziert (Weiß und Schwarzkaribisch, Weiß und Schwarzafrikanisch, Weiß und Asiatisch oder sonstiger gemischter Hintergrund).',
     '% Other':
@@ -278,7 +278,7 @@ export const details: Record<string, Record<string, string>> = {
     'Distance to nearest park (km)':
       'Luftlinienentfernung in Kilometern vom Postleitzahlenzentrum zum nächsten Parkeingang. Umfasst öffentliche Parks, Gärten, Sportplätze und Spielbereiche. Verwendet Zugangspunktstandorte aus dem OS Open Greenspace-Datensatz, sodass Immobilien an der Grenze eines großen Parks korrekt eine kurze Entfernung anzeigen.',
     'Noise (dB)':
-      'Straßenlärmpegel in Dezibel (Lden, ein 24-Stunden-gewichteter Durchschnitt) aus Defras Strategic Noise Mapping Round 4 (2022). Modelliert in 4 m Höhe über dem Boden auf einem 10-m-Raster. Über ~55 dB ist in der Regel wahrnehmbar; über ~70 dB gilt laut WHO als gesundheitsschädlich.',
+      'Straßenlärmpegel in Dezibel (Lden, ein 24-Stunden-gewichteter Durchschnitt) aus Defras Strategic Noise Mapping Round 4 (2022). Modelliert in 4 m Höhe über dem Boden auf einem 10-m-Raster. Über ~55 dB ist in der Regel wahrnehmbar; über ~70 dB gilt laut WHO als gesundheitsschädlich. Nur England; ein leerer Wert bedeutet keine kartierten Daten (nicht unbedingt leise).',
     'Max available download speed (Mbps)':
       'Maximale verfügbare Festnetz-Download-Geschwindigkeit von einem beliebigen Anbieter, aus Ofcom Connected Nations 2025. Gibt die theoretische Höchstgeschwindigkeit an, keine tatsächlich erreichten Geschwindigkeiten. 10 Mbps = Basis, 30 = schnell, 100+ = ultraschnell, 1000 = Gigabit.',
     Schools:
@@ -398,7 +398,7 @@ export const details: Record<string, Record<string, string>> = {
     '% South Asian':
       '来自2021年Census。地方政府人口中认同为印度人、巴基斯坦人、孟加拉国人或其他亚洲背景的百分比。',
     '% Black': '来自2021年Census。地方政府人口中认同为黑人、英国黑人、加勒比人或非洲人的百分比。',
-    '% East Asian': '来自2021年Census。地方政府人口中认同为华人的百分比。',
+    '% East/SE Asian': '来自2021年Census。地方政府人口中认同为华人或其他东亚/东南亚裔的百分比。',
     '% Mixed':
       '来自2021年 Census。地方政府人口中认同为混血或多种族群体（白人与黑人加勒比裔、白人与黑人非洲裔、白人与亚洲裔，或其他混血或多种族背景）的百分比。',
     '% Other':
@@ -416,7 +416,7 @@ export const details: Record<string, Record<string, string>> = {
     'Distance to nearest park (km)':
       '从邮政编码到最近公园入口的直线距离（km）。涵盖公共公园、花园、运动场和游乐场地。使用 OS Open Greenspace 数据集中的出入口位置，因此紧邻大型公园的房产可正确显示较短距离。',
     'Noise (dB)':
-      '来自Defra战略噪声图第4轮（2022年）的道路噪声水平，单位为分贝（Lden，24小时加权平均值）。在地面以上4m、10m网格间距处建模。一般而言，超过约55 dB可明显感知；超过约70 dB被世卫组织认定为有害。',
+      '来自Defra战略噪声图第4轮（2022年）的道路噪声水平，单位为分贝（Lden，24小时加权平均值）。在地面以上4m、10m网格间距处建模。一般而言，超过约55 dB可明显感知；超过约70 dB被世卫组织认定为有害。仅覆盖英格兰；空值表示无映射数据（不一定安静）。',
     'Max available download speed (Mbps)':
       '来自Ofcom Connected Nations 2025的任意运营商可提供的最大固定宽带下载速度。代表理论最大值，而非实际达到的速度。10 Mbps为基础级，30为超快级，100+为极速级，1000为千兆级。',
     Schools:
@@ -539,8 +539,8 @@ export const details: Record<string, Record<string, string>> = {
       'Census 2021 से. स्थानीय प्राधिकरण की आबादी का प्रतिशत जो खुद को भारतीय, पाकिस्तानी, बांग्लादेशी या किसी अन्य एशियाई पृष्ठभूमि के रूप में पहचानता है.',
     '% Black':
       'Census 2021 से. स्थानीय प्राधिकरण की आबादी का प्रतिशत जो खुद को अश्वेत, अश्वेत ब्रिटिश, कैरिबियाई या अफ्रीकी के रूप में पहचानता है.',
-    '% East Asian':
-      'Census 2021 से. स्थानीय प्राधिकरण की आबादी का प्रतिशत जो खुद को चीनी के रूप में पहचानता है.',
+    '% East/SE Asian':
+      'Census 2021 से. स्थानीय प्राधिकरण की आबादी का प्रतिशत जो खुद को चीनी या अन्य पूर्वी/दक्षिण-पूर्वी एशियाई के रूप में पहचानता है.',
     '% Mixed':
       'Census 2021 से. स्थानीय प्राधिकरण की आबादी का प्रतिशत जो खुद को मिश्रित या कई जातीय समूहों (श्वेत और अश्वेत कैरिबियाई, श्वेत और अश्वेत अफ्रीकी, श्वेत और एशियाई या अन्य मिश्रित/बहुजातीय पृष्ठभूमि) के रूप में पहचानता है.',
     '% Other':
@@ -562,7 +562,7 @@ export const details: Record<string, Record<string, string>> = {
     'Distance to nearest park (km)':
       'पोस्टकोड से निकटतम पार्क प्रवेश तक सीधी रेखा में दूरी, किलोमीटर में. इसमें सार्वजनिक पार्क, बगीचे, खेल मैदान और खेल स्थान शामिल हैं. OS Open Greenspace डेटा सेट के प्रवेश-बिंदु स्थानों का उपयोग करता है, इसलिए बड़े पार्क के किनारे स्थित संपत्तियां सही कम दूरी दिखाती हैं.',
     'Noise (dB)':
-      'Defra Strategic Noise Mapping Round 4 (2022) से सड़क-शोर स्तर, डेसीबल में (Lden, 24-घंटे भारित औसत). जमीन से 4 m ऊपर 10 m ग्रिड पर मॉडल किया गया. लगभग 55 dB से ऊपर शोर आमतौर पर महसूस होता है; लगभग 70 dB से ऊपर WHO इसे हानिकारक मानता है.',
+      'Defra Strategic Noise Mapping Round 4 (2022) से सड़क-शोर स्तर, डेसीबल में (Lden, 24-घंटे भारित औसत). जमीन से 4 m ऊपर 10 m ग्रिड पर मॉडल किया गया. लगभग 55 dB से ऊपर शोर आमतौर पर महसूस होता है; लगभग 70 dB से ऊपर WHO इसे हानिकारक मानता है. केवल इंग्लैंड को कवर करता है; खाली मान का अर्थ है कोई मैप किया गया डेटा नहीं (जरूरी नहीं कि शांत हो).',
     'Max available download speed (Mbps)':
       'Ofcom Connected Nations 2025 से किसी भी प्रदाता द्वारा उपलब्ध अधिकतम स्थिर ब्रॉडबैंड डाउनलोड गति. यह सैद्धांतिक अधिकतम दिखाता है, वास्तविक प्राप्त गति नहीं. 10 Mbps = बुनियादी, 30 = तेज, 100+ = अत्यंत तेज, 1000 = गीगाबिट.',
     Schools:
@@ -685,8 +685,8 @@ export const details: Record<string, Record<string, string>> = {
       'A 2021-es Census alapján. A helyi hatóság területén indiai, pakisztáni, bangladesi vagy bármely más ázsiai háttérként azonosított népesség százaléka.',
     '% Black':
       'A 2021-es Census alapján. A helyi hatóság területén fekete, brit fekete, karibi vagy afrikai háttérként azonosított népesség százaléka.',
-    '% East Asian':
-      'A 2021-es Census alapján. A helyi hatóság területén kínaiként azonosított népesség százaléka.',
+    '% East/SE Asian':
+      'A 2021-es Census alapján. A helyi hatóság területén kínai vagy más kelet-/délkelet-ázsiai származásúként azonosított népesség százaléka.',
     '% Mixed':
       'A 2021-es Census alapján. A helyi hatóság területén vegyes vagy többes etnikai csoportként (fehér és fekete karibi, fehér és fekete afrikai, fehér és ázsiai, vagy bármely más vegyes vagy többes háttér) azonosított népesség százaléka.',
     '% Other':
@@ -708,7 +708,7 @@ export const details: Record<string, Record<string, string>> = {
     'Distance to nearest park (km)':
       'Légvonalbeli távolság kilométerben az irányítószámtól a legközelebbi park bejáratáig. Magában foglalja a közparkokat, kerteket, játszótereket és szabadidős területeket. Az OS Open Greenspace adatkészlet hozzáférési pont helyszíneit használja, így a nagy park szomszédságában lévő ingatlanok helyesen rövid távolságot mutatnak.',
     'Noise (dB)':
-      'Közúti zajszint decibel (Lden, 24 órás súlyozott átlag) értékben, a Defra Stratégiai Zajtérképezés 4. fordulójából (2022). 4 m magasságban, 10 m-es rácson modellezve. ~55 dB felett általában érzékelhető; ~70 dB felett a WHO károsnak minősíti.',
+      'Közúti zajszint decibel (Lden, 24 órás súlyozott átlag) értékben, a Defra Stratégiai Zajtérképezés 4. fordulójából (2022). 4 m magasságban, 10 m-es rácson modellezve. ~55 dB felett általában érzékelhető; ~70 dB felett a WHO károsnak minősíti. Csak Angliát fedi le; az üres érték nem térképezett adatot jelent (nem feltétlenül csendes).',
     'Max available download speed (Mbps)':
       'Bármely szolgáltatótól elérhető maximális rögzített szélessávú letöltési sebesség, az Ofcom Connected Nations 2025 adataiból. Az elméleti maximumot jelöli, nem a valós sebességet. 10 Mbps = alapszintű, 30 = szupergyors, 100+ = ultragyors, 1000 = gigabites.',
     Schools:

frontend/src/i18n/locales/de.ts

@@ -214,22 +214,19 @@ const de: Translations = {
       'Relax one constraint at a time': 'Lockere eine Einschränkung nach der anderen',
       'When the search becomes too narrow, loosen a single filter and watch which postcodes reappear. This makes compromise explicit instead of relying on guesswork.':
         'Wenn die Suche zu eng wird, lockere einen einzelnen Filter und sieh, welche Postcodes wieder auftauchen. So werden Kompromisse sichtbar statt geraten.',
-      'Turn vague areas into specific postcodes':
-        'Verwandle vage Gebiete in konkrete Postcodes',
+      'Turn vague areas into specific postcodes': 'Verwandle vage Gebiete in konkrete Postcodes',
       'Broad town or borough searches hide large differences between streets. Perfect Postcode helps you move from a general area to postcodes that satisfy your hard requirements.':
         'Breite Stadt- oder Borough-Suchen verdecken große Unterschiede zwischen Straßen. Perfect Postcode hilft dir, von einem allgemeinen Gebiet zu Postcodes zu gelangen, die deine Muss-Kriterien erfüllen.',
       'Keep trade-offs visible': 'Halte Kompromisse sichtbar',
       'When there are too many or too few matches, adjust one constraint at a time and see exactly which postcodes reappear. That makes compromises explicit instead of relying on guesswork.':
         'Wenn es zu viele oder zu wenige Treffer gibt, ändere jeweils eine Anforderung und sieh genau, welche Postcodes wieder auftauchen. So werden Kompromisse sichtbar statt geraten.',
-      'Why postcode-level comparison matters':
-        'Warum Vergleiche auf Postcode-Ebene wichtig sind',
+      'Why postcode-level comparison matters': 'Warum Vergleiche auf Postcode-Ebene wichtig sind',
       'Two nearby postcodes can differ on schools, road noise, transport access, property mix, and price. Comparing at postcode level reduces the chance of treating a whole town as one uniform market.':
         'Zwei nahegelegene Postcodes können sich bei Schulen, Straßenlärm, Verkehrsanbindung, Immobilienmix und Preis stark unterscheiden. Der Vergleich auf Postcode-Ebene verhindert, dass eine ganze Stadt wie ein einheitlicher Markt behandelt wird.',
       'How to use the results': 'So nutzt du die Ergebnisse',
       'Treat matching postcodes as a research queue: check live listings, visit streets, confirm schools and admissions, and review current official sources.':
         'Behandle passende Postcodes wie eine Recherche-Liste: Prüfe Live-Inserate, geh durch die Straßen, bestätige Schulen und Admissions und nutze aktuelle offizielle Quellen.',
-      'Can I save a postcode property search?':
-        'Kann ich eine Postcode-Immobiliensuche speichern?',
+      'Can I save a postcode property search?': 'Kann ich eine Postcode-Immobiliensuche speichern?',
       'Yes. Licensed users can save searches and return to them later. Saved searches are designed for shortlists and comparison notes.':
         'Ja. Lizenzierte Nutzer können Suchen speichern und später darauf zurückgreifen. Gespeicherte Suchen sind für Auswahllisten und Vergleichsnotizen gedacht.',
       'Can I search without knowing the area?': 'Kann ich suchen, ohne die Gegend zu kennen?',
@@ -268,8 +265,7 @@ const de: Translations = {
         'Pendeln nach Postcode, nicht nur nach Ortsname',
       'Two streets in the same town can have very different station access, road routes, and public transport options. Postcode-level travel-time filtering keeps that difference visible.':
         'Zwei Straßen in derselben Stadt können sehr unterschiedliche Bahnhofsanbindungen, Straßenrouten und ÖPNV-Optionen haben. Fahrzeitfilter auf Postcode-Ebene halten diesen Unterschied sichtbar.',
-      'Balance journey time with the rest of the move':
-        'Fahrzeit mit dem restlichen Umzug abwägen',
+      'Balance journey time with the rest of the move': 'Fahrzeit mit dem restlichen Umzug abwägen',
       'A fast commute only helps if the area also fits your budget, housing needs, school preferences, safety threshold, broadband requirement, and tolerance for road noise.':
         'Ein kurzer Arbeitsweg hilft nur, wenn die Gegend auch zu Budget, Wohnbedarf, Schulwünschen, Sicherheitsgefühl, Breitbandbedarf und Lärmtoleranz passt.',
       'How travel-time filters should be interpreted': 'Wie Fahrzeitfilter zu verstehen sind',
@@ -320,8 +316,7 @@ const de: Translations = {
         'Die Schulqualität ist ein Teil der engeren Auswahl',
       'Perfect Postcode helps you compare nearby school data with the other practical constraints that shape a family move: space, price, commute, parks, safety, and local services.':
         'Mit Perfect Postcode vergleichst du Schuldaten in der Nähe mit den anderen praktischen Faktoren eines Familienumzugs: Platz, Preis, Pendelweg, Parks, Sicherheit und lokale Infrastruktur.',
-      'Check catchments before making decisions':
-        'Catchments vor Entscheidungen prüfen',
+      'Check catchments before making decisions': 'Catchments vor Entscheidungen prüfen',
       'Admissions rules and catchment boundaries can change. Use postcode-level school data to find promising areas, then verify current admissions details with the school or local authority.':
         'Admissions-Regeln und Catchment-Grenzen können sich ändern. Nutze Schuldaten auf Postcode-Ebene, um vielversprechende Gebiete zu finden, und prüfe aktuelle Details bei Schule oder Local Authority.',
       'How to treat school filters': 'Wie du Schulfilter einordnest',
@@ -474,12 +469,10 @@ const de: Translations = {
       'Make commute constraints explicit': 'Mache Pendelanforderungen klar',
       'If access to the centre, a station, hospital, university, or business park matters, use travel-time filters first and then compare the remaining postcodes by property data.':
         'Wenn die Erreichbarkeit des Zentrums, eines Bahnhofs, Krankenhauses, einer Universität oder eines Business Parks wichtig ist, nutze zuerst Fahrzeitfilter und vergleiche dann die übrigen Postcodes anhand der Immobiliendaten.',
-      'Compare value, not just headline price':
-        'Wert vergleichen, nicht nur den Angebotspreis',
+      'Compare value, not just headline price': 'Wert vergleichen, nicht nur den Angebotspreis',
       'Use price, property type, and floor-area filters together. This helps distinguish lower-cost areas from areas that simply contain smaller or different homes.':
         'Nutze Preis-, Immobilientyp- und Wohnflächenfilter gemeinsam. So kannst du günstigere Gebiete von Gebieten unterscheiden, in denen einfach kleinere oder andere Immobilien stehen.',
-      'Screen environmental and local-service signals':
-        'Umwelt- und Infrastruktur-Signale prüfen',
+      'Screen environmental and local-service signals': 'Umwelt- und Infrastruktur-Signale prüfen',
       'Road noise, parks, broadband, crime, and amenities can affect whether a property works day to day. Use them as screening criteria before booking viewings.':
         'Straßenlärm, Parks, Breitband, Kriminalität und Infrastruktur können entscheiden, ob eine Immobilie im Alltag funktioniert. Nutze sie als Screening-Kriterien, bevor du Besichtigungen buchst.',
       'Can I use this for commuter villages around Bristol?':
@@ -604,8 +597,7 @@ const de: Translations = {
     logIn: 'Anmelden',
     createAccount: 'Konto erstellen',
     resetPassword: 'Passwort zurücksetzen',
-    valueProp:
-      'Speichere Suchen, merke Immobilien und erstelle eine Shortlist passender Gebiete.',
+    valueProp: 'Speichere Suchen, merke Immobilien und erstelle eine Shortlist passender Gebiete.',
     continueWithGoogle: 'Weiter mit Google',
     email: 'E-Mail',
     emailPlaceholder: 'name@beispiel.de',
@@ -916,6 +908,7 @@ const de: Translations = {
   // ── Location Search ────────────────────────────────
   locationSearch: {
     placeholder: 'Orte oder Postcodes suchen...',
+    noResults: 'Keine passenden Orte oder Postcodes',
     postcodeNotFound: 'Postcode nicht gefunden',
     lookupFailed: 'Suche fehlgeschlagen',
     searchLabel: 'Orte oder Postcodes suchen',
@@ -1518,7 +1511,7 @@ const de: Translations = {
     '% White': '% weiß',
     '% South Asian': '% südasiatisch',
     '% Black': '% schwarz',
-    '% East Asian': '% ostasiatisch',
+    '% East/SE Asian': '% ost-/südostasiatisch',
     '% Mixed': '% gemischt',
     '% Other': '% Sonstige',
 

frontend/src/i18n/locales/en.ts

@@ -892,6 +892,7 @@ const en = {
   // ── Location Search ────────────────────────────────
   locationSearch: {
     placeholder: 'Search places or postcodes...',
+    noResults: 'No matching places or postcodes',
     postcodeNotFound: 'Postcode not found',
     lookupFailed: 'Lookup failed',
     searchLabel: 'Search places or postcodes',
@@ -1484,7 +1485,7 @@ const en = {
     '% White': '% White',
     '% South Asian': '% South Asian',
     '% Black': '% Black',
-    '% East Asian': '% East Asian',
+    '% East/SE Asian': '% East/SE Asian',
     '% Mixed': '% Mixed',
     '% Other': '% Other',
 

frontend/src/i18n/locales/fr.ts

@@ -322,8 +322,7 @@ const fr: Translations = {
         'La qualité des écoles n’est qu’une partie de la sélection',
       'Perfect Postcode helps you compare nearby school data with the other practical constraints that shape a family move: space, price, commute, parks, safety, and local services.':
         'Perfect Postcode vous aide à comparer les données des écoles à proximité avec les autres contraintes pratiques d’un déménagement familial : espace, prix, trajet, parcs, sécurité et services locaux.',
-      'Check catchments before making decisions':
-        'Vérifier les catchment areas avant de décider',
+      'Check catchments before making decisions': 'Vérifier les catchment areas avant de décider',
       'Admissions rules and catchment boundaries can change. Use postcode-level school data to find promising areas, then verify current admissions details with the school or local authority.':
         'Les règles d’admission et les limites de catchment peuvent changer. Utilisez les données scolaires par code postal pour repérer des secteurs prometteurs, puis vérifiez les admissions à jour auprès de l’école ou de l’autorité locale.',
       'How to treat school filters': 'Comment traiter les filtres scolaires',
@@ -382,8 +381,7 @@ const fr: Translations = {
         'Ce qu’une vérification du code postal ne peut pas prouver',
       'It can’t confirm the condition of a home, future development, legal title, lender requirements, or current street-level experience. Those still need direct checks.':
         'Il ne peut pas confirmer l’état d’un logement, les futurs projets, le titre de propriété, les exigences du prêteur ou le ressenti actuel dans la rue. Ces points demandent encore des vérifications directes.',
-      'Can I use the checker before a viewing?':
-        'Puis-je utiliser cet outil avant une visite ?',
+      'Can I use the checker before a viewing?': 'Puis-je utiliser cet outil avant une visite ?',
       'Yes. That’s one of the main use cases: screen the postcode first, then decide whether the viewing is worth the time.':
         'Oui. C’est l’un des principaux cas d’utilisation : examinez d’abord le code postal, puis décidez si la visite vaut le temps consacré.',
       'Does the checker include exact property condition?':
@@ -482,8 +480,7 @@ const fr: Translations = {
       'Compare value, not just headline price': 'Comparez la valeur, pas seulement le prix affiché',
       'Use price, property type, and floor-area filters together. This helps distinguish lower-cost areas from areas that simply contain smaller or different homes.':
         'Utilisez ensemble les filtres de prix, de type de bien et de surface. Cela distingue les secteurs vraiment moins chers de ceux qui comptent simplement des logements plus petits ou différents.',
-      'Screen environmental and local-service signals':
-        'Examiner environnement et services locaux',
+      'Screen environmental and local-service signals': 'Examiner environnement et services locaux',
       'Road noise, parks, broadband, crime, and amenities can affect whether a property works day to day. Use them as screening criteria before booking viewings.':
         'Le bruit routier, les parcs, le haut débit, la criminalité et les services de proximité peuvent affecter la vie quotidienne dans un logement. Utilisez-les comme critères de tri avant de réserver des visites.',
       'Can I use this for commuter villages around Bristol?':
@@ -924,6 +921,7 @@ const fr: Translations = {
   // ── Location Search ────────────────────────────────
   locationSearch: {
     placeholder: 'Rechercher des lieux ou codes postaux...',
+    noResults: 'Aucun lieu ni code postal correspondant',
     postcodeNotFound: 'Code postal introuvable',
     lookupFailed: 'Échec de la recherche',
     searchLabel: 'Rechercher des lieux ou codes postaux',
@@ -1530,7 +1528,7 @@ const fr: Translations = {
     '% White': '% Blancs',
     '% South Asian': '% Sud-Asiatiques',
     '% Black': '% Noirs',
-    '% East Asian': '% Est-Asiatiques',
+    '% East/SE Asian': '% est/sud-est asiatique',
     '% Mixed': '% Métis',
     '% Other': '% Autres',
 

frontend/src/i18n/locales/hi.ts

@@ -205,7 +205,8 @@ const hi: Translations = {
       'Relax one constraint at a time': 'एक बार में एक constraint ढीला करें',
       'When the search becomes too narrow, loosen a single filter and watch which postcodes reappear. This makes compromise explicit instead of relying on guesswork.':
         'जब search बहुत संकरी हो जाए, तो एक फ़िल्टर ढीला करें और देखें कौन से पोस्टकोड वापस आते हैं. इससे guesswork के बजाय compromise साफ दिखता है.',
-      'Turn vague areas into specific postcodes': 'धुंधले area ideas को specific postcodes में बदलें',
+      'Turn vague areas into specific postcodes':
+        'धुंधले area ideas को specific postcodes में बदलें',
       'Broad town or borough searches hide large differences between streets. Perfect Postcode helps you move from a general area to postcodes that satisfy your hard requirements.':
         'Town या borough-level searches सड़कों के बीच बड़े फर्क छिपा देती हैं. Perfect Postcode आपको general area से उन पोस्टकोड तक ले जाता है जो आपकी hard requirements पूरी करते हैं.',
       'Keep trade-offs visible': 'Trade-offs साफ रखें',
@@ -217,7 +218,8 @@ const hi: Translations = {
       'How to use the results': 'परिणामों का उपयोग कैसे करें',
       'Treat matching postcodes as a research queue: check live listings, visit streets, confirm schools and admissions, and review current official sources.':
         'Matching postcodes को research queue की तरह लें: live listings देखें, streets visit करें, schools और admissions confirm करें, और current official sources check करें.',
-      'Can I save a postcode property search?': 'क्या मैं postcode property search सेव कर सकता हूँ?',
+      'Can I save a postcode property search?':
+        'क्या मैं postcode property search सेव कर सकता हूँ?',
       'Yes. Licensed users can save searches and return to them later. Saved searches are designed for shortlists and comparison notes.':
         'हाँ. Licensed users searches सेव कर सकते हैं और बाद में वहीं लौट सकते हैं. Saved searches shortlist और comparison notes के लिए बनाई गई हैं.',
       'Can I search without knowing the area?': 'क्या area जाने बिना search कर सकता हूँ?',
@@ -231,8 +233,7 @@ const hi: Translations = {
         'Greater Manchester के आसपास broad search को narrow करने के लिए regional guide.',
       'Start a postcode search': 'Postcode search शुरू करें',
       'Commute property search': 'Commute के हिसाब से प्रॉपर्टी खोजें',
-      'Search for places to live by commute time':
-        'Commute time के हिसाब से रहने की जगहें खोजें',
+      'Search for places to live by commute time': 'Commute time के हिसाब से रहने की जगहें खोजें',
       'Commute property search - Find places to live by travel time':
         'Commute property search - travel time से रहने की जगहें खोजें',
       'Filter postcodes by commute time, then compare price, schools, safety, broadband, road noise, parks and property data on one map.':
@@ -260,8 +261,7 @@ const hi: Translations = {
         'Journey time को move की बाकी जरूरतों से balance करें',
       'A fast commute only helps if the area also fits your budget, housing needs, school preferences, safety threshold, broadband requirement, and tolerance for road noise.':
         'Fast commute तभी मदद करता है जब area आपके बजट, housing needs, school preferences, safety threshold, ब्रॉडबैंड जरूरत और road noise tolerance से भी match करता हो.',
-      'How travel-time filters should be interpreted':
-        'Travel-time filters को कैसे पढ़ें',
+      'How travel-time filters should be interpreted': 'Travel-time filters को कैसे पढ़ें',
       'Travel-time modelling is useful for comparing areas consistently. Before committing, check current timetables, disruption patterns, parking, cycling conditions, and walking routes.':
         'Travel-time modelling इलाकों की consistent comparison के लिए useful है. Commit करने से पहले current timetables, disruption patterns, parking, cycling conditions और walking routes जरूर check करें.',
       'Why commute filters are combined with property data':
@@ -305,7 +305,8 @@ const hi: Translations = {
       'Verify admissions before deciding': 'फैसले से पहले admissions verify करें',
       'School data can point to promising areas, but admissions rules and catchments can change. Confirm current arrangements with schools and local authorities.':
         'School data promising areas दिखा सकता है, लेकिन admissions rules और catchments बदल सकते हैं. Schools और local authorities से current arrangements confirm करें.',
-      'School quality is one part of the shortlist': 'School quality shortlist का सिर्फ एक हिस्सा है',
+      'School quality is one part of the shortlist':
+        'School quality shortlist का सिर्फ एक हिस्सा है',
       'Perfect Postcode helps you compare nearby school data with the other practical constraints that shape a family move: space, price, commute, parks, safety, and local services.':
         'Perfect Postcode nearby school data को उन practical constraints के साथ compare करने में मदद करता है जो family move को shape करती हैं: space, price, commute, parks, safety और local services.',
       'Check catchments before making decisions': 'फैसले से पहले catchments check करें',
@@ -317,7 +318,8 @@ const hi: Translations = {
       'Family trade-offs to compare': 'Compare करने लायक family trade-offs',
       'Combine schools with parks, road noise, crime, property size, commute, broadband, and price so the shortlist reflects the whole move.':
         'Schools को parks, road noise, crime, property size, commute, broadband और price के साथ मिलाएं ताकि shortlist पूरा move दिखाए.',
-      'Does this show school catchment guarantees?': 'क्या यह school catchment guarantee दिखाता है?',
+      'Does this show school catchment guarantees?':
+        'क्या यह school catchment guarantee दिखाता है?',
       'No. It helps identify promising areas, but catchments and admissions must be verified with the school or local authority.':
         'नहीं. यह promising areas पहचानने में मदद करता है, लेकिन catchments और admissions school या local authority से verify करने होंगे.',
       'Can I combine school filters with parks and safety?':
@@ -396,8 +398,7 @@ const hi: Translations = {
       'Compare price with property type': 'Price को property type के साथ compare करें',
       'Median prices alone can be misleading if the local property mix changes. Add property type, tenure, floor area, and price filters so similar areas are compared fairly.':
         'अगर local property mix बदलता है, तो सिर्फ median prices misleading हो सकती हैं. Similar areas की fair comparison के लिए property type, tenure, floor area और price filters जोड़ें.',
-      'Keep family and environment trade-offs visible':
-        'Family और environment trade-offs साफ रखें',
+      'Keep family and environment trade-offs visible': 'Family और environment trade-offs साफ रखें',
       'Layer school context, parks, road noise, broadband, and crime signals on top of the property filters. That makes it easier to decide which compromises are acceptable.':
         'Property filters के ऊपर school context, parks, road noise, broadband और crime signals layer करें. इससे तय करना आसान होता है कि कौन से compromises acceptable हैं.',
       'Can Perfect Postcode tell me the best area in Birmingham?':
@@ -460,8 +461,7 @@ const hi: Translations = {
       'Make commute constraints explicit': 'Commute constraints साफ करें',
       'If access to the centre, a station, hospital, university, or business park matters, use travel-time filters first and then compare the remaining postcodes by property data.':
         'अगर centre, station, hospital, university या business park तक access मायने रखती है, तो पहले travel-time filters लगाएं और फिर बचे postcodes को property data से compare करें.',
-      'Compare value, not just headline price':
-        'सिर्फ headline price नहीं, value compare करें',
+      'Compare value, not just headline price': 'सिर्फ headline price नहीं, value compare करें',
       'Use price, property type, and floor-area filters together. This helps distinguish lower-cost areas from areas that simply contain smaller or different homes.':
         'Price, property type और floor-area filters साथ इस्तेमाल करें. इससे सच में lower-cost areas को उन areas से अलग करना आसान होता है जहां बस छोटे या अलग homes हैं.',
       'Screen environmental and local-service signals':
@@ -876,6 +876,7 @@ const hi: Translations = {
 
   locationSearch: {
     placeholder: 'स्थान या पोस्टकोड खोजें...',
+    noResults: 'कोई मिलती-जुलती जगह या पोस्टकोड नहीं मिला',
     postcodeNotFound: 'पोस्टकोड नहीं मिला',
     lookupFailed: 'खोज विफल रही',
     searchLabel: 'स्थान या पोस्टकोड खोजें',
@@ -1430,7 +1431,7 @@ const hi: Translations = {
     '% White': '% श्वेत',
     '% South Asian': '% दक्षिण एशियाई',
     '% Black': '% अश्वेत',
-    '% East Asian': '% पूर्वी एशियाई',
+    '% East/SE Asian': '% पूर्वी/दक्षिण-पूर्वी एशियाई',
     '% Mixed': '% मिश्रित',
     '% Other': '% अन्य',
     'Voter turnout (%)': 'मतदाता भागीदारी (%)',

frontend/src/i18n/locales/hu.ts

@@ -471,8 +471,7 @@ const hu: Translations = {
       'Make commute constraints explicit': 'Tedd egyértelművé az ingázási korlátokat',
       'If access to the centre, a station, hospital, university, or business park matters, use travel-time filters first and then compare the remaining postcodes by property data.':
         'Ha fontos a központ, állomás, kórház, egyetem vagy business park elérése, először használd az utazási idő szűrőit, majd hasonlítsd össze a fennmaradó irányítószámokat ingatlanadatok alapján.',
-      'Compare value, not just headline price':
-        'Az értéket nézd, ne csak a kiemelt árat',
+      'Compare value, not just headline price': 'Az értéket nézd, ne csak a kiemelt árat',
       'Use price, property type, and floor-area filters together. This helps distinguish lower-cost areas from areas that simply contain smaller or different homes.':
         'Használd együtt az ár-, ingatlantípus- és alapterület-szűrőket. Ez segít megkülönböztetni az olcsóbb területeket azoktól, ahol egyszerűen kisebb vagy eltérő otthonok vannak.',
       'Screen environmental and local-service signals':
@@ -910,6 +909,7 @@ const hu: Translations = {
   // ── Location Search ────────────────────────────────
   locationSearch: {
     placeholder: 'Helyek vagy irányítószámok keresése...',
+    noResults: 'Nincs egyező hely vagy irányítószám',
     postcodeNotFound: 'Irányítószám nem található',
     lookupFailed: 'A keresés sikertelen',
     searchLabel: 'Helyek vagy irányítószámok keresése',
@@ -1511,7 +1511,7 @@ const hu: Translations = {
     '% White': '% fehér',
     '% South Asian': '% dél-ázsiai',
     '% Black': '% fekete',
-    '% East Asian': '% kelet-ázsiai',
+    '% East/SE Asian': '% kelet-/dél-kelet-ázsiai',
     '% Mixed': '% vegyes',
     '% Other': '% egyéb',
 

frontend/src/i18n/locales/zh.ts

@@ -133,7 +133,8 @@ const zh: Translations = {
         '按邮编筛选历史成交价与当前估值。',
       'Compare value with commute, schools, broadband, crime, noise, and amenities.':
         '把房价与通勤、学校、宽带、治安、噪音和周边设施放在一起比较。',
-      'Build a shortlist before spending weekends on viewings.': '把周末花在看房前，先整理出候选名单。',
+      'Build a shortlist before spending weekends on viewings.':
+        '把周末花在看房前，先整理出候选名单。',
       'Find postcodes that fit the budget before listings appear':
         '抢在房源上架之前，先锁定符合预算的邮编',
       'Start with a maximum price and property type, then colour the map by price per square metre or estimated current price. This helps reveal areas where similar homes have historically traded within reach, even when there are no live listings today.':
@@ -850,6 +851,7 @@ const zh: Translations = {
   // ── Location Search ────────────────────────────────
   locationSearch: {
     placeholder: '搜索地点或邮编...',
+    noResults: '未找到匹配的地点或邮编',
     postcodeNotFound: '未找到该邮编',
     lookupFailed: '查询失败',
     searchLabel: '搜索地点或邮编',
@@ -1073,7 +1075,8 @@ const zh: Translations = {
     dsConservationAreasUse: '英格兰指定保护区边界。用于标记邮编代表点是否位于保护区内。',
     dsListedBuildingsName: 'Historic England 受保护建筑',
     dsListedBuildingsOrigin: 'Historic England 英格兰国家遗产名录',
-    dsListedBuildingsUse: '英格兰受保护建筑点位记录。用于标记地址似乎与附近受保护建筑条目匹配的房产。',
+    dsListedBuildingsUse:
+      '英格兰受保护建筑点位记录。用于标记地址似乎与附近受保护建筑条目匹配的房产。',
     dsNaptanName: 'NaPTAN（公共交通站点）',
     dsNaptanOrigin: 'Department for Transport',
     dsNaptanUse: '英格兰各地铁路、公交、地铁/有轨电车、渡轮和机场的站点位置。',
@@ -1427,7 +1430,7 @@ const zh: Translations = {
     '% White': '% 白人',
     '% South Asian': '% 南亚裔',
     '% Black': '% 黑人',
-    '% East Asian': '% 东亚裔',
+    '% East/SE Asian': '% 东亚/东南亚裔',
     '% Mixed': '% 混血',
     '% Other': '% 其他',
 

frontend/src/lib/color-opacity.ts

@@ -0,0 +1,11 @@
+export const DEFAULT_COLOR_OPACITY = 1;
+export const MIN_COLOR_OPACITY = 0.1;
+
+export function normalizeColorOpacity(value: number | null | undefined): number {
+  if (value == null || !Number.isFinite(value)) return DEFAULT_COLOR_OPACITY;
+  return Math.min(1, Math.max(MIN_COLOR_OPACITY, value));
+}
+
+export function colorOpacityToPercent(value: number): number {
+  return Math.round(normalizeColorOpacity(value) * 100);
+}

frontend/src/lib/consts.ts

@@ -267,7 +267,7 @@ export const STACKED_GROUPS: Record<
     {
       label: 'Ethnic composition',
       unit: '%',
-      components: ['% White', '% South Asian', '% East Asian', '% Black', '% Mixed', '% Other'],
+      components: ['% White', '% South Asian', '% East/SE Asian', '% Black', '% Mixed', '% Other'],
     },
     {
       label: 'Political vote share',
@@ -384,13 +384,6 @@ export const ENUM_COLOR_OVERRIDES: Record<string, Record<string, [number, number
     F: [239, 68, 68],
     G: [126, 34, 206],
   },
-  'Max available download speed (Mbps)': {
-    '10': [107, 114, 128],
-    '30': [245, 158, 11],
-    '100': [59, 130, 246],
-    '300': [20, 184, 166],
-    '1000': [34, 197, 94],
-  },
 };
 
 /**
@@ -451,7 +444,7 @@ export const STACKED_SEGMENT_COLORS: Record<string, string> = {
   'Other crime (avg/yr)': '#6b7280',
   '% White': '#3b82f6',
   '% South Asian': '#f97316',
-  '% East Asian': '#eab308',
+  '% East/SE Asian': '#eab308',
   '% Black': '#8b5cf6',
   '% Mixed': '#14b8a6',
   '% Other': '#6b7280',

frontend/src/lib/crime-types.ts

@@ -0,0 +1,35 @@
+// Street-crime categories carried by the `crime_hotspots` vector tiles in the
+// `crime_type` feature property. The `value` strings must match the police.uk
+// "Crime type" values exactly (see pipeline/transform/crime_hotspot_tiles.py),
+// because they are used directly in the MapLibre heatmap `filter` expression.
+// `label` is a shorter, human-friendly name for the overlay-selector checkboxes.
+
+export interface CrimeTypeDef {
+  value: string;
+  label: string;
+}
+
+export const CRIME_TYPES: readonly CrimeTypeDef[] = [
+  { value: 'Violence and sexual offences', label: 'Violence & sexual offences' },
+  { value: 'Anti-social behaviour', label: 'Anti-social behaviour' },
+  { value: 'Criminal damage and arson', label: 'Criminal damage & arson' },
+  { value: 'Public order', label: 'Public order' },
+  { value: 'Shoplifting', label: 'Shoplifting' },
+  { value: 'Vehicle crime', label: 'Vehicle crime' },
+  { value: 'Burglary', label: 'Burglary' },
+  { value: 'Other theft', label: 'Other theft' },
+  { value: 'Theft from the person', label: 'Theft from the person' },
+  { value: 'Bicycle theft', label: 'Bicycle theft' },
+  { value: 'Drugs', label: 'Drugs' },
+  { value: 'Robbery', label: 'Robbery' },
+  { value: 'Possession of weapons', label: 'Possession of weapons' },
+  { value: 'Other crime', label: 'Other crime' },
+] as const;
+
+export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value);
+
+const CRIME_TYPE_VALUE_SET = new Set<string>(CRIME_TYPE_VALUES);
+
+export function isCrimeTypeValue(value: string): boolean {
+  return CRIME_TYPE_VALUE_SET.has(value);
+}

frontend/src/lib/ethnicity-filter.ts

@@ -6,7 +6,7 @@ export const ETHNICITIES_FILTER_KEY_PREFIX = `${ETHNICITIES_FILTER_NAME}:`;
 export const ETHNICITY_FEATURE_NAMES = [
   '% White',
   '% South Asian',
-  '% East Asian',
+  '% East/SE Asian',
   '% Black',
   '% Mixed',
   '% Other',

frontend/src/lib/feature-icons.tsx

@@ -367,7 +367,7 @@ const FEATURE_ICON_PATHS: Record<string, ReactNode> = {
       <path d="M16 3.13a4 4 0 010 7.75" />
     </>
   ),
-  '% East Asian': (
+  '% East/SE Asian': (
     <>
       <path d="M17 21v-2a4 4 0 00-4-4H5a4 4 0 00-4 4v2" />
       <circle cx="9" cy="7" r="4" />

pipeline/check_travel_times.py

@@ -2,9 +2,12 @@
 
 A travel-time parquet file is considered corrupted when the R5 routing
 computation failed or was interrupted, leaving either zero rows or only
-the origin postcode.  We detect this by comparing each file's row count
-against a per-mode threshold derived from the 5th-percentile of all files
-in that mode.  Files at or below 1 row are always flagged.
+the origin postcode.  We detect this by an absolute, structural criterion:
+a file is corrupt only when it is unreadable or has a row count at or below
+CORRUPT_ROW_FLOOR.  Per-mode percentile/median/range figures are reported
+for context only — they never drive the deletable set, so repeated runs
+(including with --delete) are idempotent and never erode legitimate
+small-catchment (rural/island) origins.
 
 Duplicates arise when places.parquet is rebuilt between R5 runs — each
 place gets a new numeric index prefix, so the skip-completed logic
@@ -13,7 +16,6 @@ file per slug and removes the rest.
 
 Usage:
     uv run python pipeline/check_travel_times.py [--travel-times property-data/travel-times]
-                                                  [--threshold-pct 5]
                                                   [--delete]
                                                   [--dedup]
 """
@@ -28,6 +30,12 @@ from pathlib import Path
 
 import polars as pl
 
+# Absolute row-count floor for corruption: a readable file with this many
+# rows or fewer holds at most the origin postcode (R5 failed/interrupted).
+# This is a structural threshold, NOT a population percentile, so repeated
+# runs are idempotent and never delete a fresh fraction of valid files.
+CORRUPT_ROW_FLOOR = 1
+
 
 @dataclass
 class BadFile:
@@ -74,10 +82,14 @@ def percentile(values: list[int], pct: float) -> float:
     return s[lo] + frac * (s[hi] - s[lo])
 
 
-def find_bad_files(
-    base_dir: Path, threshold_pct: float
-) -> tuple[list[BadFile], dict[str, dict]]:
-    """Scan all modes and return bad files + per-mode stats."""
+def find_bad_files(base_dir: Path) -> tuple[list[BadFile], dict[str, dict]]:
+    """Scan all modes and return bad files + per-mode stats.
+
+    A file is "bad" (deletable) only by an absolute structural criterion:
+    it is unreadable (rows < 0) or holds at most the origin postcode
+    (rows <= CORRUPT_ROW_FLOOR).  The p5/median/min/max figures are computed
+    purely for reporting and do NOT influence the deletable set.
+    """
     bad: list[BadFile] = []
     stats: dict[str, dict] = {}
 
@@ -93,15 +105,14 @@ def find_bad_files(
         if not row_counts:
             continue
 
-        p5 = percentile(row_counts, threshold_pct)
+        # Reporting statistics only — these never decide what gets deleted.
+        p5 = percentile(row_counts, 5)
         median = percentile(row_counts, 50)
-        # Threshold: max of 1 and the chosen percentile — ensures we always
-        # catch files with 0-1 rows even if p5 is 0 (e.g. walking mode).
-        threshold = max(1, int(p5))
 
         mode_bad = []
         for filename, slug, rows in entries:
-            if rows <= threshold:
+            # Corrupt = unreadable, or at/below the absolute origin-only floor.
+            if rows < 0 or rows <= CORRUPT_ROW_FLOOR:
                 bf = BadFile(mode=mode, filename=filename, slug=slug, rows=rows)
                 mode_bad.append(bf)
                 bad.append(bf)
@@ -110,7 +121,7 @@ def find_bad_files(
             "total": len(entries),
             "errors": errors,
             "bad": len(mode_bad),
-            "threshold": threshold,
+            "floor": CORRUPT_ROW_FLOOR,
             "p5": p5,
             "median": median,
             "min": min(row_counts),
@@ -169,16 +180,13 @@ def main() -> None:
         default=Path("property-data/travel-times"),
         help="Path to travel-times directory",
     )
-    parser.add_argument(
-        "--threshold-pct",
-        type=float,
-        default=5,
-        help="Percentile below which files are flagged (default: 5th)",
-    )
     parser.add_argument(
         "--delete",
         action="store_true",
-        help="Delete corrupted files (so R5 will recompute them)",
+        help=(
+            "Delete corrupted files (unreadable or "
+            f"<= {CORRUPT_ROW_FLOOR} row) so R5 will recompute them"
+        ),
     )
     parser.add_argument(
         "--dedup",
@@ -192,18 +200,20 @@ def main() -> None:
         sys.exit(1)
 
     # --- Corruption check ---
-    bad_files, stats = find_bad_files(args.travel_times, args.threshold_pct)
+    bad_files, stats = find_bad_files(args.travel_times)
 
     print("=== Per-mode summary ===\n")
+    # Floor is the absolute deletion threshold; p5/median/range are reporting
+    # context only and do not affect which files are flagged as corrupt.
     print(
-        f"{'Mode':<10} {'Total':>6} {'Bad':>5} {'Threshold':>10} {'Median':>8} {'Range':>20}"
+        f"{'Mode':<10} {'Total':>6} {'Bad':>5} {'Floor':>6} {'P5':>8} {'Median':>8} {'Range':>20}"
     )
-    print("-" * 65)
+    print("-" * 71)
     for mode, s in sorted(stats.items()):
         rng = f"{s['min']:,}–{s['max']:,}"
         print(
-            f"{mode:<10} {s['total']:>6} {s['bad']:>5} {s['threshold']:>10,} "
-            f"{s['median']:>8,.0f} {rng:>20}"
+            f"{mode:<10} {s['total']:>6} {s['bad']:>5} {s['floor']:>6,} "
+            f"{s['p5']:>8,.0f} {s['median']:>8,.0f} {rng:>20}"
         )
 
     if bad_files:

pipeline/download/arcgis.py

@@ -4,27 +4,24 @@ import polars as pl
 from pathlib import Path
 
 from pipeline.local_temp import local_tmp_dir
-from pipeline.utils import download, extract_zip
+from pipeline.utils import code_col_overrides, download, extract_zip
 
 URL = "https://www.arcgis.com/sharing/rest/content/items/36b718ad00de49afb9ad364f8b815b9e/data"
 
 
 def convert_to_parquet(data_path: Path, parquet_path: Path) -> None:
-    # Classification code columns (ruc21ind, oac11ind, imd20ind) look numeric
-    # in early rows but contain string codes like "UN1" (Unclassified) later
-    # on. Force them to String to avoid mid-stream dtype inference failures.
-    # Note: NSPL renames these year suffixes as new releases roll in (e.g.
-    # Feb 2026 bumped oac from oac21ind → oac11ind, imd from imd19ind →
-    # imd20ind), so keep this dict in sync with the current CSV headers —
-    # polars silently ignores overrides for missing columns, masking drift.
+    # Classification code columns (e.g. ruc21ind, oac11ind, imd20ind) look
+    # numeric in early rows but contain string codes like "UN1" (Unclassified)
+    # later on. Force them to String to avoid mid-stream dtype inference
+    # failures. NSPL renames these year suffixes each release, and polars
+    # silently ignores overrides for missing columns, so match on the
+    # suffix-free stem (read from the header) rather than hard-coding suffixes.
+    csv_path = data_path / "Data/NSPL_FEB_2026_UK.csv"
+    names = pl.scan_csv(csv_path).collect_schema().names()
     df = pl.scan_csv(
-        data_path / "Data/NSPL_FEB_2026_UK.csv",
+        csv_path,
         try_parse_dates=True,
-        schema_overrides={
-            "ruc21ind": pl.String,
-            "oac11ind": pl.String,
-            "imd20ind": pl.String,
-        },
+        schema_overrides=code_col_overrides(names),
     )
     print(f"Columns: {df.collect_schema().names()}")
     parquet_path.parent.mkdir(parents=True, exist_ok=True)

pipeline/download/broadband.py

@@ -51,8 +51,16 @@ def _obtain_zip(dest: Path) -> None:
 
 
 def convert_to_parquet(extract_dir: Path, parquet_path: Path) -> None:
-    # Find CSV files in the extracted directory
-    csv_files = list(extract_dir.rglob("*.csv"))
+    # Find CSV files in the extracted directory. The zip ships two sibling
+    # dirs with identical headers: postcode_files/ (all premises) and
+    # postcode_res_files/ (residential only). Take all-premises only so each
+    # postcode_space appears once; matching on the dir name keeps this
+    # resilient to the date-stamped top-level folder prefix.
+    csv_files = [
+        f
+        for f in extract_dir.rglob("*.csv")
+        if "postcode_res_files" not in f.parts
+    ]
     if not csv_files:
         raise FileNotFoundError(f"No CSV files found in {extract_dir}")
 

pipeline/download/ethnicity.py

@@ -36,7 +36,8 @@ GEOGRAPHY_CODE_REPLACEMENTS = {
 
 def _ethnicity_percentages(df: pl.DataFrame) -> pl.DataFrame:
     # Use the detailed 19+1 breakdown to get sub-categories for Asian ethnicity,
-    # then aggregate back to the broad groups plus South Asian / East Asian split.
+    # then aggregate back to the broad groups plus a South Asian / East/SE Asian
+    # split (Indian/Pakistani/Bangladeshi vs Chinese + other East/SE Asian).
     detailed = df.filter(
         (pl.col("Ethnicity_type") == "ONS 2021 19+1") & (pl.col("Ethnicity") != "All")
     )
@@ -53,9 +54,13 @@ def _ethnicity_percentages(df: pl.DataFrame) -> pl.DataFrame:
         "Indian": "South Asian",
         "Pakistani": "South Asian",
         "Bangladeshi": "South Asian",
-        "Any Other Asian Background": "South Asian",
-        # East Asian
-        "Chinese": "East Asian",
+        # East / Southeast Asian. The ONS "Any Other Asian Background" bucket is
+        # predominantly East/Southeast Asian (Filipino, Vietnamese, Thai,
+        # Japanese, Korean, ...) rather than South Asian, so route it here rather
+        # than inflating "% South Asian". The split is approximate (the ONS
+        # bucket also holds some South Asian groups such as Sri Lankan/Nepalese).
+        "Chinese": "East/SE Asian",
+        "Any Other Asian Background": "East/SE Asian",
         # Black
         "Black African": "Black",
         "Black Caribbean": "Black",

pipeline/download/inspire.py

@@ -4,7 +4,10 @@ Downloads GML files for all local authorities from the INSPIRE download page.
 Each ZIP contains a GML file with title extent polygons for that authority.
 
 Source: https://use-land-property-data.service.gov.uk/datasets/inspire/download
-License: INSPIRE End User Licence
+License: Open Government Licence v3.0 (since 1 July 2020, under the PSGA).
+    Requires HM Land Registry + Ordnance Survey (AC0000851063) attribution; see
+    the conditions page at the source URL. Boundaries are indicative "general
+    boundaries", not the legal extent of title.
 """
 
 import argparse

pipeline/download/median_age.py

@@ -43,6 +43,47 @@ AGE_BANDS = [
     (85, 5),  # Aged 85 years and over
 ]
 
+# Canonical NOMIS TS007A (C2021_AGE_19_NAME) band labels, in the SAME order as
+# AGE_BANDS. Index i here corresponds to AGE_BANDS[i]; we validate the pivot
+# output against this set and use it (not positional string parsing) to order
+# the columns, so a stray/relabelled/missing band fails loudly instead of
+# silently mis-aligning counts against the wrong lower bound.
+EXPECTED_BAND_NAMES = [
+    "Aged 0 to 4 years",
+    "Aged 5 to 9 years",
+    "Aged 10 to 14 years",
+    "Aged 15 to 19 years",
+    "Aged 20 to 24 years",
+    "Aged 25 to 29 years",
+    "Aged 30 to 34 years",
+    "Aged 35 to 39 years",
+    "Aged 40 to 44 years",
+    "Aged 45 to 49 years",
+    "Aged 50 to 54 years",
+    "Aged 55 to 59 years",
+    "Aged 60 to 64 years",
+    "Aged 65 to 69 years",
+    "Aged 70 to 74 years",
+    "Aged 75 to 79 years",
+    "Aged 80 to 84 years",
+    "Aged 85 years and over",
+]
+assert len(EXPECTED_BAND_NAMES) == len(AGE_BANDS), (
+    "EXPECTED_BAND_NAMES and AGE_BANDS must stay aligned 1:1"
+)
+
+# NOMIS sometimes labels a band with a wording variant that denotes the SAME
+# age range (e.g. "Aged 4 years and under" for ages 0-4, "Aged 90 years and
+# over" wording for the top band). Map such known-equivalent labels back to the
+# canonical name BEFORE validation so a real band change still fails loudly,
+# but a cosmetic relabel of an identical range does not block the build.
+BAND_NAME_ALIASES = {
+    "Aged 4 years and under": "Aged 0 to 4 years",
+}
+assert set(BAND_NAME_ALIASES.values()) <= set(EXPECTED_BAND_NAMES), (
+    "BAND_NAME_ALIASES must map to canonical EXPECTED_BAND_NAMES"
+)
+
 
 def compute_median_age(counts: list[int]) -> float:
     """Compute median age from five-year band counts using linear interpolation."""
@@ -62,6 +103,62 @@ def compute_median_age(counts: list[int]) -> float:
     return float("nan")
 
 
+def _bands_to_median_table(pivoted: pl.DataFrame) -> pl.DataFrame:
+    """Validate the pivoted age-band columns, then compute median age per LSOA.
+
+    The pivot must contain exactly the canonical NOMIS TS007A bands; a
+    missing/extra/relabelled band would otherwise silently mis-align counts
+    against the wrong AGE_BANDS lower bound, so we fail loudly instead.
+    """
+    # Normalise known-equivalent NOMIS label variants to their canonical name
+    # before validating (renaming onto an already-present canonical column would
+    # collide, so polars raises loudly in that genuinely ambiguous case).
+    rename_map = {
+        c: BAND_NAME_ALIASES[c] for c in pivoted.columns if c in BAND_NAME_ALIASES
+    }
+    if rename_map:
+        pivoted = pivoted.rename(rename_map)
+
+    # Validate the pivoted age-band columns against the canonical NOMIS set
+    # BEFORE computing anything.
+    band_cols = [c for c in pivoted.columns if c != "GEOGRAPHY_CODE"]
+    found = set(band_cols)
+    expected = set(EXPECTED_BAND_NAMES)
+    if found != expected:
+        missing = sorted(expected - found)
+        unexpected = sorted(found - expected)
+        raise ValueError(
+            "Census age-band columns do not match the expected NOMIS TS007A bands.\n"
+            f"  expected {len(EXPECTED_BAND_NAMES)} bands, found {len(band_cols)}\n"
+            f"  missing:    {missing}\n"
+            f"  unexpected: {unexpected}\n"
+            "Refusing to compute medians against misaligned bands."
+        )
+
+    # Use the canonical order (guaranteed aligned with AGE_BANDS), not positional
+    # string parsing, and treat a null band (zero-population) as 0 rather than
+    # crashing on sum().
+    band_cols = list(EXPECTED_BAND_NAMES)
+    pivoted = pivoted.with_columns(pl.col(band_cols).fill_null(0))
+
+    print(f"Age bands found: {len(band_cols)}")
+    print(f"  First: {band_cols[0]}")
+    print(f"  Last:  {band_cols[-1]}")
+
+    rows = pivoted.select("GEOGRAPHY_CODE", *band_cols).to_dicts()
+    medians = []
+    for row in rows:
+        counts = [row[col] for col in band_cols]
+        median = compute_median_age(counts)
+        medians.append(
+            {"lsoa21": row["GEOGRAPHY_CODE"], "median_age": round(median, 1)}
+        )
+
+    return pl.DataFrame(medians).with_columns(
+        pl.col("median_age").cast(pl.Float32),
+    )
+
+
 def download_and_convert(output_path: Path) -> None:
     print("Downloading Census 2021 age by five-year bands from NOMIS...")
     frames = []
@@ -94,29 +191,7 @@ def download_and_convert(output_path: Path) -> None:
         values="OBS_VALUE",
     )
 
-    # Extract age band columns in order and compute median
-    # NOMIS returns band names like "Aged 0 to 4 years", "Aged 85 years and over"
-    band_cols = [c for c in pivoted.columns if c != "GEOGRAPHY_CODE"]
-    # Sort by the lower bound of each band
-    band_cols.sort(key=lambda c: int(c.split()[1]))
-
-    print(f"Age bands found: {len(band_cols)}")
-    print(f"  First: {band_cols[0]}")
-    print(f"  Last:  {band_cols[-1]}")
-
-    # Compute median age per LSOA
-    rows = pivoted.select("GEOGRAPHY_CODE", *band_cols).to_dicts()
-    medians = []
-    for row in rows:
-        counts = [row[col] for col in band_cols]
-        median = compute_median_age(counts)
-        medians.append(
-            {"lsoa21": row["GEOGRAPHY_CODE"], "median_age": round(median, 1)}
-        )
-
-    result = pl.DataFrame(medians).with_columns(
-        pl.col("median_age").cast(pl.Float32),
-    )
+    result = _bands_to_median_table(pivoted)
 
     print(f"England LSOAs: {result.height}")
     print(

pipeline/download/naptan.py

@@ -17,7 +17,12 @@ TUBE_STATION_MERGE_RADIUS_DEGREES = 0.01
 
 STOP_TYPES = {
     "AIR": "Airport",
+    # Ferry: FER/FBT are the terminal/berth nodes; FTD is a docking entrance.
+    "FER": "Ferry",
+    "FBT": "Ferry",
     "FTD": "Ferry",
+    # Rail: RLY is the station node; RSE is a station entrance.
+    "RLY": "Rail station",
     "RSE": "Rail station",
     "BCT": "Bus stop",
     "BCE": "Bus station",
@@ -26,6 +31,16 @@ STOP_TYPES = {
     "MET": "Tube station",
 }
 
+# Stop types that are access/entrance nodes rather than the primary station or
+# terminal node. During dedup the primary node (e.g. RLY/FER) wins so a station
+# with both a station node and entrances yields one POI at the station node.
+ENTRANCE_STOP_TYPES = {"RSE", "FTD"}
+
+# Categories whose entrances/variants are merged into a single station-level POI
+# by normalized name + area (like Tube stations), so an RLY node and its RSE
+# entrances collapse to one POI at the station node.
+STATION_MERGE_CATEGORIES = {TUBE_STATION_CATEGORY, "Rail station", "Ferry"}
+
 
 OUTPUT_COLUMNS = ["id", "name", "category", "lat", "lng"]
 
@@ -97,7 +112,9 @@ def _empty_output_frame() -> pl.DataFrame:
     )
 
 
-def station_name_score(name: str) -> tuple[int, int]:
+def station_name_score(name: str, entrance: bool = False) -> tuple[int, int, int]:
+    # Prefer the primary station/terminal node over an entrance, then a name
+    # without a transport-mode suffix, then the shorter name.
     lower = name.lower()
     suffix_penalty = int(
         lower.endswith(
@@ -112,7 +129,7 @@ def station_name_score(name: str) -> tuple[int, int]:
             )
         )
     )
-    return (suffix_penalty, len(name))
+    return (int(entrance), suffix_penalty, len(name))
 
 
 @dataclass
@@ -122,6 +139,7 @@ class StationAccumulator:
     category: str
     lat_sum: float
     lng_sum: float
+    entrance: bool = False
     count: int = 1
 
     @property
@@ -143,9 +161,13 @@ class StationAccumulator:
         self.count += 1
 
         name = str(row["name"] or "")
-        if station_name_score(name) < station_name_score(self.name):
+        entrance = bool(row.get("entrance"))
+        if station_name_score(name, entrance) < station_name_score(
+            self.name, self.entrance
+        ):
             self.id = str(row["id"] or "")
             self.name = name
+            self.entrance = entrance
 
 
 def _station_from_row(row: dict[str, object]) -> StationAccumulator:
@@ -155,19 +177,23 @@ def _station_from_row(row: dict[str, object]) -> StationAccumulator:
         category=str(row["category"] or ""),
         lat_sum=float(row["lat"]),
         lng_sum=float(row["lng"]),
+        entrance=bool(row.get("entrance")),
     )
 
 
-def _deduplicate_tube_stations(df: pl.DataFrame) -> pl.DataFrame:
+def _deduplicate_station_areas(df: pl.DataFrame) -> pl.DataFrame:
     if len(df) == 0:
         return _empty_output_frame()
 
     selected: list[StationAccumulator] = []
-    groups: dict[str, list[int]] = {}
+    groups: dict[tuple[str, str], list[int]] = {}
 
     for row in df.iter_rows(named=True):
-        station_key = canonical_station_name(str(row["name"] or ""))
-        if not station_key:
+        # Key by category so different modes sharing a name/area (e.g. a rail
+        # station and a ferry terminal) are not merged into one POI.
+        category = str(row["category"] or "")
+        station_key = (category, canonical_station_name(str(row["name"] or "")))
+        if not station_key[1]:
             selected.append(_station_from_row(row))
             continue
 
@@ -198,7 +224,7 @@ def _deduplicate_tube_stations(df: pl.DataFrame) -> pl.DataFrame:
     ).select(OUTPUT_COLUMNS)
 
 
-def _deduplicate_non_tube_stops(df: pl.DataFrame) -> pl.DataFrame:
+def _deduplicate_local_stops(df: pl.DataFrame) -> pl.DataFrame:
     if len(df) == 0:
         return _empty_output_frame()
 
@@ -218,7 +244,10 @@ def _deduplicate_non_tube_stops(df: pl.DataFrame) -> pl.DataFrame:
             .select(OUTPUT_COLUMNS)
         )
     if len(no_loc) > 0:
-        frames.append(no_loc.select(OUTPUT_COLUMNS))
+        # Stops with no locality can't be deduped by locality, so merge genuine
+        # co-located duplicates (same name+category within the same small area)
+        # via the station-area logic, while keeping distinct far-apart stops.
+        frames.append(_deduplicate_station_areas(no_loc))
 
     if not frames:
         return _empty_output_frame()
@@ -227,14 +256,20 @@ def _deduplicate_non_tube_stops(df: pl.DataFrame) -> pl.DataFrame:
 
 
 def deduplicate_naptan(df: pl.DataFrame) -> pl.DataFrame:
-    """Deduplicate NaPTAN stops, with station-level merging for Tube POIs."""
-    tube = df.filter(pl.col("category") == TUBE_STATION_CATEGORY)
-    other = df.filter(pl.col("category") != TUBE_STATION_CATEGORY)
+    """Deduplicate NaPTAN stops, merging station/terminal entrances by area.
+
+    Tube, rail and ferry POIs are merged to one record per station by
+    normalized name + area, with the primary station/terminal node (e.g. RLY,
+    FER) winning over an entrance node (RSE, FTD). Other stops are deduplicated
+    by exact name+category+locality.
+    """
+    station = df.filter(pl.col("category").is_in(list(STATION_MERGE_CATEGORIES)))
+    other = df.filter(~pl.col("category").is_in(list(STATION_MERGE_CATEGORIES)))
 
     return pl.concat(
         [
-            _deduplicate_non_tube_stops(other),
-            _deduplicate_tube_stations(tube),
+            _deduplicate_local_stops(other),
+            _deduplicate_station_areas(station),
         ]
     ).select(OUTPUT_COLUMNS)
 
@@ -263,6 +298,7 @@ def download_naptan(output: Path) -> None:
             pl.col("Latitude").alias("lat"),
             pl.col("Longitude").alias("lng"),
             pl.col("NptgLocalityCode").alias("locality"),
+            pl.col("StopType").is_in(list(ENTRANCE_STOP_TYPES)).alias("entrance"),
         )
     )
 

pipeline/download/noise.py

@@ -83,11 +83,32 @@ NATIVE_RESOLUTION = 10
 # Request pixel resolution in metres.
 RESOLUTION = NATIVE_RESOLUTION
 
+# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
+# otherwise inside the raster means "modelled below the lowest reporting band",
+# i.e. genuinely quiet — see noise_overlay_tiles.py:167.
+NOISE_NODATA_SENTINEL = np.float32(-96.0)
+
+# Lowest modelled Defra Lden reporting band (dB). Verified against the actual
+# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
+# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor
+# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
+# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
+# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
+# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
+# NOISE_COLOR_STOPS[0]) — a rendering threshold, not the data's reporting floor.
+NOISE_QUIET_FLOOR_DB = np.float32(40.0)
+
 # The pipeline has postcode representative points rather than complete unit
 # polygons here. Use a small local footprint and take the maximum 10m cell so
 # postcode-level noise is not understated by centroid rounding.
 POSTCODE_NOISE_RADIUS_M = 50
 
+# Adjacent download tiles must overlap by at least the sampling radius so every
+# postcode's 50m max-window is fully contained in at least one tile. Without
+# this, a loud pixel just across a tile seam is invisible to a postcode on the
+# far side, under-reporting noise near seams.
+TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
+
 # Retry/split behaviour for slow Defra WCS requests. Some 100km eastern tiles
 # intermittently return 504s; smaller fallback requests usually succeed.
 MAX_RETRIES = 3
@@ -245,8 +266,12 @@ def _download_tile(
         except (
             NoGeoTiffError,
             httpx.HTTPStatusError,
-            httpx.TimeoutException,
-            httpx.ConnectError,
+            # TransportError is the superset of TimeoutException, ConnectError,
+            # ReadError and ProtocolError — including RemoteProtocolError, raised
+            # when the WCS server closes the connection mid-stream ("incomplete
+            # chunked read"). All are transient; retry/split rather than letting
+            # one flaky tile crash the whole raster download.
+            httpx.TransportError,
         ) as e:
             last_error = e
             if attempt < MAX_RETRIES:
@@ -287,6 +312,31 @@ def _download_tile(
     return [], [(min_e, min_n, max_e, max_n)]
 
 
+def _generate_tiles(
+    min_e: int,
+    max_e: int,
+    min_n: int,
+    max_n: int,
+    tile_size: int,
+    overlap_m: int,
+    step: int,
+) -> list[Tile]:
+    """Generate download tile bboxes stepping by ``step`` but extending each
+    tile's far edge by ``overlap_m`` so neighbours overlap.
+
+    Overlapping neighbours guarantee that every postcode's POSTCODE_NOISE_RADIUS_M
+    sampling window is fully contained in at least one tile, so a loud pixel near
+    a seam is never lost (the sampler takes np.fmax across tiles).
+    """
+    tiles: list[Tile] = []
+    for tile_min_e in range(min_e, max_e, step):
+        for tile_min_n in range(min_n, max_n, step):
+            tile_max_e = min(tile_min_e + tile_size + overlap_m, BNG_MAX_E)
+            tile_max_n = min(tile_min_n + tile_size + overlap_m, BNG_MAX_N)
+            tiles.append((tile_min_e, tile_min_n, tile_max_e, tile_max_n))
+    return tiles
+
+
 def download_raster(
     tile_dir: Path,
     wcs_base: str,
@@ -296,12 +346,9 @@ def download_raster(
     allow_missing_tiles: bool = False,
 ) -> list[Path]:
     """Download noise GeoTIFF raster covering England, returning paths to saved files."""
-    tiles = []
-    for min_e in range(BNG_MIN_E, BNG_MAX_E, TILE_SIZE):
-        for min_n in range(BNG_MIN_N, BNG_MAX_N, TILE_SIZE):
-            max_e = min(min_e + TILE_SIZE, BNG_MAX_E)
-            max_n = min(min_n + TILE_SIZE, BNG_MAX_N)
-            tiles.append((min_e, min_n, max_e, max_n))
+    tiles = _generate_tiles(
+        BNG_MIN_E, BNG_MAX_E, BNG_MIN_N, BNG_MAX_N, TILE_SIZE, TILE_OVERLAP_M, TILE_SIZE
+    )
 
     print(
         f"[{label}] Downloading {len(tiles)} tiles at {RESOLUTION}m resolution ({MAX_WORKERS} workers)..."
@@ -385,14 +432,23 @@ def sample_noise_at_postcodes(
             if len(candidate_indices) == 0:
                 continue
 
+            # Defra rasters encode TRUE nodata as the -96.0 sentinel (and
+            # occasionally non-finite / dataset.nodata); genuinely quiet ground
+            # below the model's lowest reporting band is encoded as 0.0. Only
+            # the former is "we don't know" — the latter is a real "we know it's
+            # quiet" reading and must not collapse to null. So treat ONLY true
+            # nodata as -inf (it never wins a max and never counts as coverage),
+            # and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
+            # below-threshold 0.0 surfaces as the documented quiet floor.
             grid = dataset.read(1).astype(np.float32, copy=False)
-            invalid = ~np.isfinite(grid) | (grid == 0)
+            nodata = ~np.isfinite(grid) | np.isclose(
+                grid, NOISE_NODATA_SENTINEL, rtol=1e-5, atol=1e-5
+            )
             if dataset.nodata is not None:
-                invalid |= np.isclose(
+                nodata |= np.isclose(
                     grid, np.float32(dataset.nodata), rtol=1e-5, atol=1e-5
                 )
-            grid = grid.copy()
-            grid[invalid] = -np.inf
+            grid = np.where(nodata, -np.inf, np.maximum(grid, NOISE_QUIET_FLOOR_DB))
             if filter_size > 1:
                 grid = maximum_filter(
                     grid, size=filter_size, mode="constant", cval=-np.inf
@@ -412,12 +468,15 @@ def sample_noise_at_postcodes(
 
             sampled_indices = candidate_indices[in_bounds]
             sampled = grid[rows[in_bounds], cols[in_bounds]]
-            valid = sampled != -np.inf
-            if not np.any(valid):
+            # A finite sample means at least one in-coverage cell sat in the
+            # window (quiet -> floor, or louder). -inf means the whole window was
+            # true nodata, so the postcode stays uncovered (null) for this tile.
+            covered = np.isfinite(sampled)
+            if not np.any(covered):
                 continue
 
-            sampled_indices = sampled_indices[valid]
-            sampled = sampled[valid]
+            sampled_indices = sampled_indices[covered]
+            sampled = sampled[covered]
             existing = noise_db[sampled_indices]
             noise_db[sampled_indices] = np.where(
                 np.isnan(existing), sampled, np.maximum(existing, sampled)

pipeline/download/ofsted.py

@@ -1,4 +1,5 @@
 import argparse
+import io
 import tempfile
 import polars as pl
 from pathlib import Path
@@ -14,10 +15,12 @@ URL = "https://assets.publishing.service.gov.uk/media/69c5269b4a06660f0854427b/M
 def convert_to_parquet(csv_path: Path, parquet_path: Path) -> None:
     print("Reading CSV...")
 
+    # The gov.uk source is cp1252-encoded; decode explicitly so non-ASCII
+    # school names are not corrupted (see gias.py for the same approach).
+    text = csv_path.read_bytes().decode("cp1252")
     df = pl.read_csv(
-        csv_path,
+        io.StringIO(text),
         infer_schema_length=10000,
-        encoding="utf8-lossy",
         null_values=["NULL", "Not applicable"],
     )
 

pipeline/download/places.py

@@ -42,6 +42,41 @@ SEARCH_PLACE_TYPES = {
     "island",
 }
 TRAVEL_DESTINATION_PLACE_TYPES = {"city"}
+
+# Named OSM highways worth surfacing as searchable streets (N). Service roads, footways,
+# cycleways and motorways are deliberately excluded.
+SEARCHABLE_HIGHWAY_TYPES = {
+    "residential",
+    "unclassified",
+    "tertiary",
+    "tertiary_link",
+    "secondary",
+    "secondary_link",
+    "primary",
+    "primary_link",
+    "trunk",
+    "living_street",
+    "pedestrian",
+}
+
+# High-value named POIs (M) lifted from uk_pois.parquet into the gazetteer, mapped from the
+# OSM "key/value" category onto a search place_type. Everyday shops/amenities are excluded.
+HIGH_VALUE_POI_CATEGORIES = {
+    "leisure/park": "park",
+    "leisure/garden": "park",
+    "leisure/nature_reserve": "park",
+    "leisure/common": "park",
+    "tourism/attraction": "attraction",
+    "tourism/theme_park": "attraction",
+    "tourism/zoo": "attraction",
+    "tourism/museum": "attraction",
+    "tourism/gallery": "attraction",
+    "amenity/hospital": "hospital",
+    "healthcare/hospital": "hospital",
+    "shop/mall": "retail",
+    "shop/department_store": "retail",
+}
+
 ENGLAND_COUNTRY_CODE = "E92000001"
 LONDON_REGION_CODE = "E12000007"
 LONDON_LAD_PREFIX = "E09"
@@ -49,6 +84,38 @@ LONDON_COUNTY_CODES = {"E13000001", "E13000002"}
 DISPLAY_CITY_NEAREST_POSTCODE_MAX_M = 3_000
 WGS84_TO_BNG = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
 
+# England British National Grid (EPSG:27700) bounding box, with margin. ONS NSPL stores
+# postcodes that have no grid reference at the Null-Island sentinel lat=99.999999,
+# long=0.000000, whose paired easting/northing collapse to the grid origin (0, 0) (or
+# inf). Requiring coordinates inside this box drops the sentinel from every index, so an
+# active postcode lacking a grid ref can never become a false nearest neighbour.
+ENGLAND_BNG_MIN_EAST = 50_000.0
+ENGLAND_BNG_MAX_EAST = 660_000.0
+ENGLAND_BNG_MIN_NORTH = 0.0
+ENGLAND_BNG_MAX_NORTH = 660_000.0
+
+
+def _valid_wgs84_expr() -> pl.Expr:
+    """Rows with a real lat/long inside England (drops the ONS lat=99.999999, long=0.0
+    no-grid-reference sentinel and any nulls), so they never enter a coordinate index."""
+    return (
+        pl.col("lat").is_not_null()
+        & pl.col("long").is_not_null()
+        & pl.col("lat").is_between(ENGLAND_BBOX_SOUTH, ENGLAND_BBOX_NORTH)
+        & pl.col("long").is_between(ENGLAND_BBOX_WEST, ENGLAND_BBOX_EAST)
+    )
+
+
+def _valid_bng_expr() -> pl.Expr:
+    """Rows with a real easting/northing inside England (drops the (0, 0) grid-origin /
+    inf paired with the ONS no-grid-reference sentinel and any nulls)."""
+    return (
+        pl.col("east1m").is_not_null()
+        & pl.col("north1m").is_not_null()
+        & pl.col("east1m").is_between(ENGLAND_BNG_MIN_EAST, ENGLAND_BNG_MAX_EAST)
+        & pl.col("north1m").is_between(ENGLAND_BNG_MIN_NORTH, ENGLAND_BNG_MAX_NORTH)
+    )
+
 # Suffixes to strip from raw station names before appending the typed suffix.
 _STATION_STRIP = (
     " tube station",
@@ -240,11 +307,165 @@ def _slugify_name(name: str) -> str:
     return re.sub(r"\s+", "-", slug).strip("-")
 
 
+def _street_centroid(coords: list[tuple[float, float]]) -> tuple[float, float] | None:
+    """Average (lat, lon) of a way's vertices."""
+    if not coords:
+        return None
+    count = len(coords)
+    lat = sum(lat for lat, _ in coords) / count
+    lon = sum(lon for _, lon in coords) / count
+    return lat, lon
+
+
+def _normalize_street_name(name: str) -> str:
+    """Grouping key for a street name: collapse whitespace, lowercase."""
+    return re.sub(r"\s+", " ", name).strip().lower()
+
+
+def _outcode_of_postcode(postcode: str) -> str:
+    """Outward code (everything before the space) of a postcode, e.g. 'NW1' from 'NW1 6XE'."""
+    return postcode.split(" ", 1)[0] if postcode else ""
+
+
+def _outcode_tree(postcodes_path: Path) -> tuple[cKDTree, list[str]]:
+    """Build a nearest-neighbour index from postcode coordinates to their outcode, so each
+    street can be tagged with the outcode it sits in (used to disambiguate same-named roads)."""
+    df = (
+        pl.read_parquet(
+            postcodes_path, columns=["pcds", "lat", "long", "ctry25cd", "doterm"]
+        )
+        .filter((pl.col("ctry25cd") == ENGLAND_COUNTRY_CODE) & pl.col("doterm").is_null())
+        .filter(_valid_wgs84_expr())
+    )
+    coords = np.column_stack(
+        [df["lat"].to_numpy().astype(np.float64), df["long"].to_numpy().astype(np.float64)]
+    )
+    outcodes = [_outcode_of_postcode(pc) for pc in df["pcds"].to_list()]
+    return cKDTree(coords), outcodes
+
+
+def _build_street_places(
+    streets: list[dict],
+    tree: cKDTree,
+    outcodes: list[str],
+) -> list[dict]:
+    """Group street segments by (normalized name, outcode), averaging centroids, so a road that
+    OSM splits into many segments becomes one searchable result per outcode it passes through."""
+    if not streets:
+        return []
+
+    coords = np.array([[street["lat"], street["lon"]] for street in streets], dtype=np.float64)
+    _, indices = tree.query(coords)
+
+    grouped: dict[tuple[str, str], dict] = {}
+    for street, postcode_idx in zip(streets, indices):
+        outcode = outcodes[postcode_idx]
+        key = (_normalize_street_name(street["name"]), outcode)
+        entry = grouped.get(key)
+        if entry is None:
+            grouped[key] = {
+                "name": street["name"],
+                "lat_sum": street["lat"],
+                "lon_sum": street["lon"],
+                "count": 1,
+            }
+        else:
+            entry["lat_sum"] += street["lat"]
+            entry["lon_sum"] += street["lon"]
+            entry["count"] += 1
+
+    places = []
+    for entry in grouped.values():
+        count = entry["count"]
+        places.append(
+            {
+                "name": entry["name"],
+                "place_type": "street",
+                "lat": entry["lat_sum"] / count,
+                "lon": entry["lon_sum"] / count,
+                "population": 0,
+                "travel_destination": False,
+                "display_city": None,
+            }
+        )
+    return sorted(places, key=lambda place: place["name"].lower())
+
+
+def _poi_dedup_key(name: str, place_type: str, lat: float, lon: float) -> tuple:
+    """Geographic de-dup key: round(.,2) is ~1.1km lat / ~0.7km UK lon.
+
+    Coarse enough to collapse the SAME physical POI mapped twice a few metres
+    apart, fine enough to keep genuinely distinct same-named POIs in different
+    towns (e.g. "Victoria Park" in London vs Bristol).
+    """
+    return (name.lower(), place_type, round(lat, 2), round(lon, 2))
+
+
+def _pois_to_places(pois: pl.DataFrame) -> list[dict]:
+    """Map high-value named POIs onto gazetteer place rows (M), de-duplicated by (name, type, coords)."""
+    if pois.is_empty():
+        return []
+
+    seen: set[tuple] = set()
+    places: list[dict] = []
+    for row in pois.iter_rows(named=True):
+        place_type = HIGH_VALUE_POI_CATEGORIES.get(str(row.get("category", "")))
+        if place_type is None:
+            continue
+        name = str(row.get("name") or "").strip()
+        if len(name) < 3:
+            continue
+        lat = float(row["lat"])
+        lon = float(row["lng"])
+        key = _poi_dedup_key(name, place_type, lat, lon)
+        if key in seen:
+            continue
+        seen.add(key)
+        places.append(
+            {
+                "name": name,
+                "place_type": place_type,
+                "lat": lat,
+                "lon": lon,
+                "population": 0,
+                "travel_destination": False,
+                "display_city": None,
+            }
+        )
+    return places
+
+
+def _append_high_value_pois(places: list[dict], pois_path: Path) -> int:
+    pois = pl.read_parquet(pois_path, columns=["name", "category", "lat", "lng"])
+    new_places = _pois_to_places(pois)
+    existing = {
+        _poi_dedup_key(
+            str(place["name"]), place["place_type"], place["lat"], place["lon"]
+        )
+        for place in places
+    }
+    added = 0
+    for place in new_places:
+        key = _poi_dedup_key(
+            place["name"], place["place_type"], place["lat"], place["lon"]
+        )
+        if key in existing:
+            continue
+        places.append(place)
+        existing.add(key)
+        added += 1
+    return added
+
+
 def _postcode_lookup(postcodes_path: Path) -> dict[str, tuple[float, float]]:
-    df = pl.read_parquet(
+    df = (
+        pl.read_parquet(
             postcodes_path,
             columns=["pcds", "lat", "long", "ctry25cd", "doterm"],
-    ).filter((pl.col("ctry25cd") == ENGLAND_COUNTRY_CODE) & pl.col("doterm").is_null())
+        )
+        .filter((pl.col("ctry25cd") == ENGLAND_COUNTRY_CODE) & pl.col("doterm").is_null())
+        .filter(_valid_wgs84_expr())
+    )
     return {
         _normalize_postcode(postcode): (float(lat), float(lon))
         for postcode, lat, lon in df.select(["pcds", "lat", "long"]).iter_rows()
@@ -266,6 +487,19 @@ def _display_city_from_tags(tags: dict[str, str]) -> str | None:
     return None
 
 
+def _parse_population(pop_str: str) -> int:
+    """Robustly parse OSM population tags that may carry grouping separators,
+    decimals, or surrounding text ("12,345", "5 000", "12345.0", "approx 5000").
+    """
+    # Take the integer part before any decimal point, then the first run of
+    # digits ignoring grouping separators (commas/spaces) and other annotations.
+    match = re.search(r"\d[\d,\s]*", pop_str.split(".", 1)[0])
+    if match is None:
+        return 0
+    digits = re.sub(r"\D", "", match.group(0))
+    return int(digits) if digits else 0
+
+
 def _is_london_admin_expr() -> pl.Expr:
     return (
         (pl.col("rgn25cd") == LONDON_REGION_CODE)
@@ -289,7 +523,7 @@ def _london_postcode_tree(postcodes_path: Path) -> tuple[cKDTree, np.ndarray]:
         .filter(
             (pl.col("ctry25cd") == ENGLAND_COUNTRY_CODE) & pl.col("doterm").is_null()
         )
-        .filter(pl.col("east1m").is_not_null() & pl.col("north1m").is_not_null())
+        .filter(_valid_bng_expr())
         .with_columns(_is_london_admin_expr().alias("is_london"))
         .select("east1m", "north1m", "is_london")
     )
@@ -466,11 +700,15 @@ def _append_naptan_dlr_stations(places: list[dict], naptan_path: Path) -> int:
 
 
 class PlaceHandler(osmium.SimpleHandler):
-    def __init__(self, progress: tqdm, england_polygon) -> None:
+    def __init__(
+        self, progress: tqdm, england_polygon, *, collect_streets: bool = False
+    ) -> None:
         super().__init__()
         self._progress = progress
         self.places: list[dict] = []
+        self.streets: list[dict] = []
         self._england = england_polygon
+        self._collect_streets = collect_streets
 
     def _add(
         self,
@@ -513,11 +751,7 @@ class PlaceHandler(osmium.SimpleHandler):
         if not name:
             return
 
-        pop_str = tags.get("population", "")
-        try:
-            population = int(pop_str)
-        except ValueError:
-            population = 0
+        population = _parse_population(tags.get("population", ""))
 
         # place=* nodes
         place_type = tags.get("place")
@@ -551,6 +785,39 @@ class PlaceHandler(osmium.SimpleHandler):
             )
             return
 
+    def way(self, w: osmium.osm.Way) -> None:
+        """Collect named, searchable highways as raw segments (grouped into streets later)."""
+        if not self._collect_streets:
+            return
+        self._progress.update(1)
+        if w.tags.get("highway") not in SEARCHABLE_HIGHWAY_TYPES:
+            return
+        name = w.tags.get("name:en", w.tags.get("name", ""))
+        if not name:
+            return
+
+        # Way node refs expose resolved .lat/.lon directly (locations=True); accessing them
+        # raises InvalidLocationError when a node's location is missing from the index.
+        coords: list[tuple[float, float]] = []
+        for node in w.nodes:
+            try:
+                coords.append((node.lat, node.lon))
+            except osmium.InvalidLocationError:
+                continue
+
+        centroid = _street_centroid(coords)
+        if centroid is None:
+            return
+        lat, lon = centroid
+        if not (
+            ENGLAND_BBOX_SOUTH <= lat <= ENGLAND_BBOX_NORTH
+            and ENGLAND_BBOX_WEST <= lon <= ENGLAND_BBOX_EAST
+        ):
+            return
+        if not self._england.contains(Point(lon, lat)):
+            return
+        self.streets.append({"name": name, "lat": lat, "lon": lon})
+
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Extract place names from OSM PBF")
@@ -578,15 +845,28 @@ def main() -> None:
         "--postcodes",
         type=Path,
         help=(
-            "Postcode parquet used to geocode OfS university contact postcodes "
-            "and assign Greater London display labels"
+            "Postcode parquet used to geocode OfS university contact postcodes, assign "
+            "Greater London display labels, and tag streets with their outcode"
         ),
     )
+    parser.add_argument(
+        "--pois",
+        type=Path,
+        help="Optional uk_pois.parquet; high-value named POIs are added to the gazetteer",
+    )
+    parser.add_argument(
+        "--include-streets",
+        action="store_true",
+        help="Extract named highways as searchable streets (requires --postcodes)",
+    )
     args = parser.parse_args()
 
     pbf_file = args.pbf
     england_polygon = load_england_polygon(args.boundary)
 
+    if args.include_streets and not args.postcodes:
+        raise ValueError("--postcodes is required with --include-streets")
+
     print("Extracting search place nodes + railway stations")
     with tqdm(
         unit=" elements",
@@ -595,10 +875,21 @@ def main() -> None:
         smoothing=0.05,
         mininterval=1.0,
     ) as progress:
-        handler = PlaceHandler(progress, england_polygon)
+        handler = PlaceHandler(
+            progress, england_polygon, collect_streets=args.include_streets
+        )
         handler.apply_file(str(pbf_file), locations=True)
 
     print(f"Extracted {len(handler.places):,} place nodes")
+    if args.include_streets:
+        print(f"Collected {len(handler.streets):,} named street segments")
+        tree, outcodes = _outcode_tree(args.postcodes)
+        street_places = _build_street_places(handler.streets, tree, outcodes)
+        handler.places.extend(street_places)
+        print(f"Added {len(street_places):,} grouped streets")
+    if args.pois:
+        added = _append_high_value_pois(handler.places, args.pois)
+        print(f"Added {added:,} high-value POIs from {args.pois}")
     if args.naptan:
         added = _append_naptan_dlr_stations(handler.places, args.naptan)
         print(f"Added {added:,} DLR station destinations from NaPTAN")

pipeline/download/satellite_highres.py

@@ -0,0 +1,503 @@
+"""Build a high-resolution England aerial PMTiles archive from EA Vertical Aerial Photography.
+
+The Environment Agency / Defra Vertical Aerial Photography (VAP) archive is open
+(OGL v3.0) RGB orthophotography at 10-50 cm, distributed as 5 km ECW tiles on the
+British National Grid. There is no public imagery tile service, so we mirror the
+Sentinel-2 ``satellite.pmtiles`` approach: query the Defra survey download API for
+an area of interest, pick the best RGB capture per OS tile, download and decode the
+ECW rasters, re-tile them into Web-Mercator raster tiles, and bake a single PMTiles
+archive that the server stacks *over* the Sentinel-2 base where coverage exists.
+
+ECW decoding needs a GDAL build that includes the (free, read-only) ERDAS ECW/JP2
+SDK, which is not present in the rasterio wheel. The mosaic + tiling step therefore
+runs inside a GDAL-with-ECW Docker image (see ``docker/gdal-ecw/Dockerfile``); the
+rest of the pipeline is plain Python plus the ``pmtiles`` CLI.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shutil
+import sqlite3
+import subprocess
+import tempfile
+import urllib.error
+import urllib.request
+import zipfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from pathlib import Path
+
+from pipeline.download.tiles import ensure_pmtiles_cli
+from pipeline.local_temp import local_tmp_dir
+
+# Defra Data Services Platform survey download API (reverse-engineered from the
+# environment.data.gov.uk/survey front-end; no official API is documented).
+SEARCH_URL = (
+    "https://environment.data.gov.uk/backend/catalog/api/tiles/collections/survey/search"
+)
+SURVEY_PAGE_URL = "https://environment.data.gov.uk/survey"
+# Static public key baked into the survey page JS. May rotate -- we try to scrape a
+# fresh one from the page and only fall back to this literal.
+DEFAULT_SUBSCRIPTION_KEY = "dspui"
+SUBSCRIPTION_KEY_RE = re.compile(r"subscription-key=([A-Za-z0-9]+)")
+
+# True-colour RGB product only (skip IRRGB near-infra-red and Night Time variants).
+VAP_RGB_PRODUCT = "vertical_aerial_photography_tiles_rgb"
+
+# Greater London bounding box (lon/lat). The API only returns tiles where coverage
+# exists, so a generous bbox is fine -- it does not force blank downloads.
+DEFAULT_AOI: dict = {
+    "type": "Polygon",
+    "coordinates": [
+        [
+            [-0.55, 51.25],
+            [0.30, 51.25],
+            [0.30, 51.70],
+            [-0.55, 51.70],
+            [-0.55, 51.25],
+        ]
+    ],
+}
+
+DEFAULT_MIN_ZOOM = 14
+DEFAULT_MAX_ZOOM = 19
+# GDAL image with the ECW driver. The official OSGeo image does not ship ECW, so
+# this defaults to the locally-built image from docker/gdal-ecw/Dockerfile.
+DEFAULT_GDAL_IMAGE = "perfect-postcode/gdal-ecw:latest"
+USER_AGENT = "perfect-postcode-satellite-highres/1.0"
+ATTRIBUTION_TEMPLATE = (
+    "Environment Agency Vertical Aerial Photography - "
+    "© Environment Agency copyright and/or database right {year}. "
+    "All rights reserved. Licensed under the Open Government Licence v3.0."
+)
+
+
+@dataclass(frozen=True)
+class VapTile:
+    """One survey download record from the Defra search API."""
+
+    product_id: str
+    year: int
+    resolution_m: float
+    os_tile_id: str
+    uri: str
+    label: str
+
+
+def parse_search_results(payload: dict) -> list[VapTile]:
+    """Turn a raw search-API JSON payload into typed records."""
+    tiles: list[VapTile] = []
+    for result in payload.get("results", []):
+        try:
+            tiles.append(
+                VapTile(
+                    product_id=result["product"]["id"],
+                    year=int(result["year"]["id"]),
+                    resolution_m=float(result["resolution"]["id"]),
+                    os_tile_id=result["tile"]["id"],
+                    uri=result["uri"],
+                    label=result.get("label", ""),
+                )
+            )
+        except (KeyError, TypeError, ValueError):
+            # Skip malformed records rather than failing the whole search.
+            continue
+    return tiles
+
+
+def select_best_rgb_tiles(tiles: list[VapTile]) -> list[VapTile]:
+    """Pick one RGB capture per OS tile: finest resolution, then latest year.
+
+    Pure function -- the unit test exercises this against a real-shaped payload.
+    """
+    best: dict[str, VapTile] = {}
+    for tile in tiles:
+        if tile.product_id != VAP_RGB_PRODUCT:
+            continue
+        current = best.get(tile.os_tile_id)
+        if current is None or _is_better(tile, current):
+            best[tile.os_tile_id] = tile
+    return [best[key] for key in sorted(best)]
+
+
+def _is_better(candidate: VapTile, incumbent: VapTile) -> bool:
+    """Finer resolution wins; ties broken by the most recent survey year."""
+    if candidate.resolution_m != incumbent.resolution_m:
+        return candidate.resolution_m < incumbent.resolution_m
+    return candidate.year > incumbent.year
+
+
+def _http_get(url: str, timeout: float) -> bytes:
+    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(req, timeout=timeout) as response:
+        return response.read()
+
+
+def resolve_subscription_key(explicit: str | None, timeout: float = 30.0) -> str:
+    """Use an explicit key, else scrape the survey page JS, else the known default."""
+    if explicit:
+        return explicit
+    try:
+        page = _http_get(SURVEY_PAGE_URL, timeout).decode("utf-8", "ignore")
+        match = SUBSCRIPTION_KEY_RE.search(page)
+        if match:
+            return match.group(1)
+        # The key usually lives in a referenced JS chunk; scan the largest one.
+        for chunk in re.findall(r'src="(/_next/static/[^"]+\.js)"', page):
+            js = _http_get(f"https://environment.data.gov.uk{chunk}", timeout)
+            match = SUBSCRIPTION_KEY_RE.search(js.decode("utf-8", "ignore"))
+            if match:
+                return match.group(1)
+    except (urllib.error.URLError, TimeoutError, ConnectionError) as err:
+        print(f"Could not scrape subscription key ({err}); using default", flush=True)
+    return DEFAULT_SUBSCRIPTION_KEY
+
+
+def search_vap_tiles(aoi: dict, timeout: float = 60.0) -> list[VapTile]:
+    """POST the area-of-interest polygon and return the RGB tiles to download."""
+    body = json.dumps(aoi).encode("utf-8")
+    req = urllib.request.Request(
+        SEARCH_URL,
+        data=body,
+        headers={
+            "Content-Type": "application/geo+json",
+            "Referer": SURVEY_PAGE_URL,
+            "User-Agent": USER_AGENT,
+        },
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as response:
+        payload = json.load(response)
+    selected = select_best_rgb_tiles(parse_search_results(payload))
+    print(
+        f"Search returned {payload.get('count', 0)} records; "
+        f"selected {len(selected)} RGB tile(s)",
+        flush=True,
+    )
+    return selected
+
+
+def _download_and_extract(
+    tile: VapTile, ecw_dir: Path, key: str, timeout: float, retries: int
+) -> list[Path]:
+    """Download one survey zip and extract its ECW raster(s)."""
+    url = f"{tile.uri}?subscription-key={key}"
+    zip_path = ecw_dir / f"{tile.os_tile_id}.zip"
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(
+                urllib.request.Request(url, headers={"User-Agent": USER_AGENT}),
+                timeout=timeout,
+            ) as response, zip_path.open("wb") as out:
+                shutil.copyfileobj(response, out, length=1 << 20)
+            break
+        except (urllib.error.URLError, TimeoutError, ConnectionError) as err:
+            if attempt == retries:
+                raise RuntimeError(f"Failed to download {url}: {err}") from err
+    extracted: list[Path] = []
+    with zipfile.ZipFile(zip_path) as archive:
+        for member in archive.infolist():
+            if member.is_dir() or not member.filename.lower().endswith(".ecw"):
+                continue
+            target = ecw_dir / f"{tile.os_tile_id}_{Path(member.filename).name}"
+            with archive.open(member) as src, target.open("wb") as dst:
+                shutil.copyfileobj(src, dst, length=1 << 20)
+            extracted.append(target)
+    zip_path.unlink(missing_ok=True)
+    if not extracted:
+        print(f"  {tile.os_tile_id}: no ECW in archive (skipped)", flush=True)
+    return extracted
+
+
+def download_tiles(
+    tiles: list[VapTile],
+    ecw_dir: Path,
+    key: str,
+    max_workers: int,
+    timeout: float,
+    retries: int,
+) -> list[Path]:
+    """Download every selected tile concurrently; return all extracted ECW paths."""
+    ecw_dir.mkdir(parents=True, exist_ok=True)
+    ecw_paths: list[Path] = []
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(
+                _download_and_extract, tile, ecw_dir, key, timeout, retries
+            ): tile
+            for tile in tiles
+        }
+        done = 0
+        for future in as_completed(futures):
+            tile = futures[future]
+            ecw_paths.extend(future.result())
+            done += 1
+            print(
+                f"Downloaded {done}/{len(tiles)} tiles "
+                f"(latest: {tile.os_tile_id} {tile.resolution_m}m {tile.year})",
+                flush=True,
+            )
+    return ecw_paths
+
+
+def _build_tiles_with_gdal(
+    work_dir: Path,
+    gdal_image: str,
+    min_zoom: int,
+    max_zoom: int,
+    jobs: int,
+    webp_quality: int,
+) -> Path:
+    """Mosaic the ECW rasters and emit XYZ WebP tiles inside the GDAL-with-ECW image.
+
+    Returns the host path of the generated ``xyz`` directory. We use lossy WebP with
+    an alpha channel: ~6x smaller than lossless PNG for photographic imagery while
+    keeping transparency, so coverage gaps stay see-through and the Sentinel-2 base
+    shows through them.
+    """
+    xyz_dir = work_dir / "xyz"
+    # EA "RGB" ECWs are 4-band RGBA (band 4 is a constant-255 validity/alpha mask),
+    # so we build a plain 4-band VRT (no -addalpha, which would make a 5th band and
+    # exceed PNG's 4-band limit). We then:
+    #   * force EPSG:27700 -- the pixels are already British National Grid, and the
+    #     EPSG code lets PROJ apply the OSTN15 datum shift (grid ships in the image)
+    #     for metre-accurate reprojection to Web Mercator;
+    #   * label band 4 as alpha so gdal2tiles writes transparent PNGs. Inter-block
+    #     gaps the VRT fills with 0 then read as alpha=0 (transparent), letting the
+    #     Sentinel-2 base show through wherever VAP coverage is missing.
+    script = (
+        "set -euo pipefail; "
+        "cd /work; "
+        "gdalbuildvrt -resolution highest mosaic.vrt ecw/*.ecw; "
+        "gdal_edit.py -a_srs EPSG:27700 "
+        "-colorinterp_1 red -colorinterp_2 green -colorinterp_3 blue "
+        "-colorinterp_4 alpha mosaic.vrt; "
+        f"gdal2tiles.py --xyz --zoom={min_zoom}-{max_zoom} "
+        f"--processes={jobs} --resampling=average --webviewer=none "
+        f"--tiledriver=WEBP --webp-quality={webp_quality} "
+        "mosaic.vrt xyz"
+    )
+    subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{work_dir.resolve()}:/work",
+            gdal_image,
+            "bash",
+            "-c",
+            script,
+        ],
+        check=True,
+    )
+    if not xyz_dir.exists():
+        raise RuntimeError("gdal2tiles produced no output directory")
+    return xyz_dir
+
+
+def _pack_xyz_to_mbtiles(
+    xyz_dir: Path,
+    mbtiles_path: Path,
+    bounds: tuple[float, float, float, float],
+    min_zoom: int,
+    max_zoom: int,
+    attribution: str,
+) -> int:
+    """Pack a gdal2tiles XYZ WebP directory into an MBTiles SQLite file (TMS rows)."""
+    if mbtiles_path.exists():
+        mbtiles_path.unlink()
+    conn = sqlite3.connect(mbtiles_path)
+    try:
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        conn.execute("CREATE TABLE metadata (name TEXT, value TEXT)")
+        conn.execute(
+            "CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, "
+            "tile_row INTEGER, tile_data BLOB)"
+        )
+        conn.execute(
+            "CREATE UNIQUE INDEX tile_index ON tiles "
+            "(zoom_level, tile_column, tile_row)"
+        )
+        conn.executemany(
+            "INSERT INTO metadata (name, value) VALUES (?, ?)",
+            [
+                ("name", "EA Vertical Aerial Photography"),
+                ("type", "overlay"),
+                ("version", "1"),
+                ("description", "Environment Agency high-resolution aerial imagery"),
+                ("format", "webp"),
+                ("attribution", attribution),
+                ("bounds", ",".join(f"{value:.6f}" for value in bounds)),
+                ("minzoom", str(min_zoom)),
+                ("maxzoom", str(max_zoom)),
+            ],
+        )
+        inserted = 0
+        for zoom_dir in sorted(xyz_dir.iterdir()):
+            if not zoom_dir.is_dir() or not zoom_dir.name.isdigit():
+                continue
+            zoom = int(zoom_dir.name)
+            for col_dir in zoom_dir.iterdir():
+                if not col_dir.is_dir() or not col_dir.name.isdigit():
+                    continue
+                col = int(col_dir.name)
+                for tile_file in col_dir.glob("*.webp"):
+                    if not tile_file.stem.isdigit():
+                        continue
+                    row = int(tile_file.stem)
+                    tms_row = (1 << zoom) - 1 - row
+                    conn.execute(
+                        "INSERT OR REPLACE INTO tiles VALUES (?, ?, ?, ?)",
+                        (zoom, col, tms_row, tile_file.read_bytes()),
+                    )
+                    inserted += 1
+                    if inserted % 5000 == 0:
+                        conn.commit()
+                        print(f"  packed {inserted:,} tiles", flush=True)
+        conn.commit()
+    finally:
+        conn.close()
+    return inserted
+
+
+def build_satellite_highres_tiles(
+    output_path: Path,
+    pmtiles_bin: Path,
+    pmtiles_version: str,
+    aoi: dict,
+    min_zoom: int,
+    max_zoom: int,
+    gdal_image: str,
+    subscription_key: str | None,
+    max_workers: int,
+    timeout: float,
+    retries: int,
+    jobs: int,
+    webp_quality: int,
+) -> None:
+    if min_zoom > max_zoom:
+        raise ValueError("--min-zoom must be <= --max-zoom")
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    ensure_pmtiles_cli(pmtiles_bin, pmtiles_version)
+
+    tiles = search_vap_tiles(aoi)
+    if not tiles:
+        raise RuntimeError("No RGB Vertical Aerial Photography tiles for the AOI")
+    key = resolve_subscription_key(subscription_key)
+    attribution = ATTRIBUTION_TEMPLATE.format(year=max(tile.year for tile in tiles))
+
+    with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as tmp:
+        work_dir = Path(tmp)
+        ecw_dir = work_dir / "ecw"
+        ecw_paths = download_tiles(
+            tiles, ecw_dir, key, max_workers, timeout, retries
+        )
+        if not ecw_paths:
+            raise RuntimeError("No ECW rasters were extracted from the downloads")
+
+        xyz_dir = _build_tiles_with_gdal(
+            work_dir, gdal_image, min_zoom, max_zoom, jobs, webp_quality
+        )
+
+        mbtiles_path = work_dir / "satellite_highres.mbtiles"
+        bounds = _aoi_bounds(aoi)
+        inserted = _pack_xyz_to_mbtiles(
+            xyz_dir, mbtiles_path, bounds, min_zoom, max_zoom, attribution
+        )
+        if inserted == 0:
+            raise RuntimeError("Tiling produced no tiles to pack")
+        print(f"Packed {inserted:,} tiles into MBTiles", flush=True)
+
+        subprocess.run(
+            [str(pmtiles_bin), "convert", str(mbtiles_path), str(output_path), "--force"],
+            check=True,
+        )
+
+    size_mb = output_path.stat().st_size / (1024 * 1024)
+    print(f"Wrote {output_path} ({size_mb:.1f} MB) -- {attribution}", flush=True)
+
+
+def _aoi_bounds(aoi: dict) -> tuple[float, float, float, float]:
+    coords = [point for ring in aoi["coordinates"] for point in ring]
+    lons = [point[0] for point in coords]
+    lats = [point[1] for point in coords]
+    return min(lons), min(lats), max(lons), max(lats)
+
+
+def _load_aoi(path: Path | None) -> dict:
+    if path is None:
+        return DEFAULT_AOI
+    data = json.loads(path.read_text())
+    if data.get("type") == "FeatureCollection":
+        return data["features"][0]["geometry"]
+    if data.get("type") == "Feature":
+        return data["geometry"]
+    return data
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--pmtiles-bin", type=Path, default=Path("property-data/pmtiles"))
+    parser.add_argument("--pmtiles-version", default="1.22.3")
+    parser.add_argument(
+        "--aoi-geojson",
+        type=Path,
+        default=None,
+        help="GeoJSON Polygon/Feature/FeatureCollection for the area of interest "
+        "(default: Greater London)",
+    )
+    parser.add_argument("--min-zoom", type=int, default=DEFAULT_MIN_ZOOM)
+    parser.add_argument("--max-zoom", type=int, default=DEFAULT_MAX_ZOOM)
+    parser.add_argument(
+        "--gdal-image",
+        default=DEFAULT_GDAL_IMAGE,
+        help="Docker image with a GDAL that has the ECW driver",
+    )
+    parser.add_argument(
+        "--subscription-key",
+        default=None,
+        help="Override the Defra survey API key (default: scrape, then 'dspui')",
+    )
+    parser.add_argument("--max-workers", type=int, default=4)
+    parser.add_argument("--timeout", type=float, default=600.0)
+    parser.add_argument("--retries", type=int, default=3)
+    parser.add_argument(
+        "--jobs",
+        type=int,
+        default=8,
+        help="Parallel processes for gdal2tiles",
+    )
+    parser.add_argument(
+        "--webp-quality",
+        type=int,
+        default=85,
+        help="WebP tile quality (1-100); lower is smaller",
+    )
+    args = parser.parse_args()
+
+    build_satellite_highres_tiles(
+        output_path=args.output,
+        pmtiles_bin=args.pmtiles_bin,
+        pmtiles_version=args.pmtiles_version,
+        aoi=_load_aoi(args.aoi_geojson),
+        min_zoom=args.min_zoom,
+        max_zoom=args.max_zoom,
+        gdal_image=args.gdal_image,
+        subscription_key=args.subscription_key,
+        max_workers=max(1, args.max_workers),
+        timeout=args.timeout,
+        retries=max(0, args.retries),
+        jobs=max(1, args.jobs),
+        webp_quality=args.webp_quality,
+    )
+
+
+if __name__ == "__main__":
+    main()

pipeline/download/test_ethnicity.py

@@ -35,3 +35,31 @@ def test_ethnicity_percentages_recombines_predecessor_lads_by_population():
     assert cumberland.select("% White", "% South Asian").to_dicts() == [
         {"% White": 45.0, "% South Asian": 55.0}
     ]
+
+
+def test_ethnicity_routes_any_other_asian_to_east_se_asian():
+    """'Any Other Asian Background' and 'Chinese' both fold into '% East/SE Asian'
+    (not '% South Asian'), fixing the East/SE Asian undercount."""
+    rows = [
+        {
+            "Geography_code": "E06000001",
+            "Ethnicity_type": "ONS 2021 19+1",
+            "Ethnicity": ethnicity,
+            "Ethnic Population": pop,
+            "Value1": 0.0,
+        }
+        for ethnicity, pop in [
+            ("Chinese", 30),
+            ("Any Other Asian Background", 20),
+            ("Indian", 50),
+        ]
+    ]
+
+    result = _ethnicity_percentages(pl.DataFrame(rows))
+    area = result.filter(pl.col("Geography_code") == "E06000001")
+
+    assert "% East/SE Asian" in result.columns
+    assert "% East Asian" not in result.columns
+    assert area.select("% East/SE Asian", "% South Asian").to_dicts() == [
+        {"% East/SE Asian": 50.0, "% South Asian": 50.0}
+    ]

pipeline/download/test_median_age.py

@@ -0,0 +1,90 @@
+import math
+
+import polars as pl
+import pytest
+
+from pipeline.download import median_age
+from pipeline.download.median_age import (
+    AGE_BANDS,
+    EXPECTED_BAND_NAMES,
+    compute_median_age,
+)
+
+
+def test_expected_band_names_align_with_age_bands():
+    assert len(EXPECTED_BAND_NAMES) == len(AGE_BANDS)
+
+
+def test_compute_median_age_interpolates_within_median_band():
+    # All weight in the 30-34 band -> median is the band midpoint via linear
+    # interpolation: 30 + ((50 - 0) / 100) * 5 = 32.5.
+    counts = [0] * len(AGE_BANDS)
+    counts[6] = 100  # "Aged 30 to 34 years"
+    assert compute_median_age(counts) == pytest.approx(32.5)
+
+    # 50 below the median band, 100 inside the 35-39 band holding the median.
+    # half = 75; cumulative before band 7 = 50; 35 + ((75 - 50) / 100) * 5 = 36.25.
+    counts = [0] * len(AGE_BANDS)
+    counts[0] = 50  # below the median band
+    counts[7] = 100  # "Aged 35 to 39 years" holds the median
+    assert compute_median_age(counts) == pytest.approx(36.25)
+
+
+def test_compute_median_age_empty_lsoa_is_nan():
+    assert math.isnan(compute_median_age([0] * len(AGE_BANDS)))
+
+
+def _pivoted(band_to_counts: dict[str, list]) -> pl.DataFrame:
+    """Build a pivot-shaped frame: GEOGRAPHY_CODE + one column per band."""
+    n = len(next(iter(band_to_counts.values())))
+    data = {"GEOGRAPHY_CODE": [f"E0100000{i}" for i in range(n)]}
+    data.update(band_to_counts)
+    return pl.DataFrame(data)
+
+
+def test_null_band_count_is_treated_as_zero_not_crash():
+    # One LSOA has a null in the 85+ band (NOMIS can return null for a band with
+    # zero people). It must be coerced to 0, not raise TypeError in sum(). With
+    # all 100 people in the 30-34 band the median is the band midpoint, 32.5.
+    counts_by_band = {name: [0] for name in EXPECTED_BAND_NAMES}
+    counts_by_band["Aged 30 to 34 years"] = [100]
+    counts_by_band["Aged 85 years and over"] = [None]
+    pivoted = _pivoted(counts_by_band)
+
+    table = median_age._bands_to_median_table(pivoted)
+
+    assert table.height == 1
+    assert table["median_age"][0] == pytest.approx(32.5)
+
+
+def test_equivalent_band_label_alias_is_accepted():
+    # NOMIS relabelled the first band "Aged 4 years and under" (same as ages
+    # 0-4). It must be normalised to the canonical name and used as band 0, not
+    # rejected. All 100 people in that band -> median in the 0-4 range: 2.5.
+    counts_by_band = {name: [0] for name in EXPECTED_BAND_NAMES}
+    counts_by_band["Aged 4 years and under"] = counts_by_band.pop("Aged 0 to 4 years")
+    counts_by_band["Aged 4 years and under"] = [100]
+    pivoted = _pivoted(counts_by_band)
+
+    table = median_age._bands_to_median_table(pivoted)
+
+    assert table.height == 1
+    assert table["median_age"][0] == pytest.approx(2.5)
+
+
+def test_missing_band_raises_clear_error():
+    counts_by_band = {name: [10] for name in EXPECTED_BAND_NAMES}
+    del counts_by_band["Aged 85 years and over"]
+    pivoted = _pivoted(counts_by_band)
+
+    with pytest.raises(ValueError, match=r"do not match the expected NOMIS"):
+        median_age._bands_to_median_table(pivoted)
+
+
+def test_relabelled_band_raises_clear_error():
+    counts_by_band = {name: [10] for name in EXPECTED_BAND_NAMES}
+    counts_by_band["Total"] = counts_by_band.pop("Aged 85 years and over")
+    pivoted = _pivoted(counts_by_band)
+
+    with pytest.raises(ValueError, match=r"unexpected:"):
+        median_age._bands_to_median_table(pivoted)

pipeline/download/test_naptan.py

@@ -89,3 +89,92 @@ def test_deduplicate_naptan_does_not_merge_missing_locality_bus_stops():
     result = deduplicate_naptan(df)
 
     assert len(result) == 2
+
+
+def test_deduplicate_naptan_merges_colocated_missing_locality_bus_stations():
+    # Two NaPTAN records for the same bus station with no locality, co-located
+    # within the merge area, are a true duplicate and collapse to one POI.
+    df = pl.DataFrame(
+        {
+            "id": ["a", "b"],
+            "name": ["Victoria Bus Station", "Victoria Bus Station"],
+            "category": ["Bus station", "Bus station"],
+            "lat": [51.4952, 51.4953],
+            "lng": [-0.1441, -0.1440],
+            "locality": [None, None],
+        }
+    )
+
+    result = deduplicate_naptan(df)
+
+    assert len(result) == 1
+    assert result["name"][0] == "Victoria Bus Station"
+    assert result["category"][0] == "Bus station"
+    assert result["lat"][0] == pytest.approx((51.4952 + 51.4953) / 2)
+
+
+def test_deduplicate_naptan_keeps_rail_station_with_only_station_node():
+    # Aberdare's only NaPTAN record is an RLY station node (StopType "RLY").
+    df = pl.DataFrame(
+        {
+            "id": ["aberdare-rly"],
+            "name": ["Aberdare Rail Station"],
+            "category": ["Rail station"],
+            "lat": [51.7155],
+            "lng": [-3.4438],
+            "locality": ["ABERDARE"],
+            "entrance": [False],
+        }
+    )
+
+    result = deduplicate_naptan(df)
+
+    assert len(result) == 1
+    assert result["name"][0] == "Aberdare Rail Station"
+    assert result["category"][0] == "Rail station"
+
+
+def test_deduplicate_naptan_merges_rail_entrances_into_station_node():
+    # A station node (RLY) and its two entrance nodes (RSE) collapse to a single
+    # "Rail station" POI represented by the station node, not an entrance.
+    df = pl.DataFrame(
+        {
+            "id": ["clapham-rly", "clapham-rse-a", "clapham-rse-b"],
+            "name": [
+                "Clapham Junction Rail Station",
+                "Clapham Junction Rail Station",
+                "Clapham Junction Rail Station",
+            ],
+            "category": ["Rail station", "Rail station", "Rail station"],
+            "lat": [51.4642, 51.4644, 51.4640],
+            "lng": [-0.1705, -0.1702, -0.1708],
+            "locality": ["CLAPHAM", "CLAPHAM", "CLAPHAM"],
+            "entrance": [False, True, True],
+        }
+    )
+
+    result = deduplicate_naptan(df)
+
+    assert len(result) == 1
+    assert result["id"][0] == "clapham-rly"
+    assert result["category"][0] == "Rail station"
+
+
+def test_deduplicate_naptan_does_not_merge_rail_and_ferry_in_same_area():
+    # Different transport modes sharing a name/area stay as separate POIs.
+    df = pl.DataFrame(
+        {
+            "id": ["harbour-rail", "harbour-ferry"],
+            "name": ["Harbour Station", "Harbour Station"],
+            "category": ["Rail station", "Ferry"],
+            "lat": [51.5, 51.5001],
+            "lng": [-0.1, -0.1001],
+            "locality": ["HARBOUR", "HARBOUR"],
+            "entrance": [False, False],
+        }
+    )
+
+    result = deduplicate_naptan(df).sort("category")
+
+    assert len(result) == 2
+    assert result["category"].to_list() == ["Ferry", "Rail station"]

pipeline/download/test_noise.py

@@ -2,14 +2,32 @@ import httpx
 import numpy as np
 import pytest
 import rasterio
+from rasterio.io import MemoryFile
 from rasterio.transform import from_origin
 
 from pipeline.download import noise
 
 
+def _tiny_geotiff_bytes() -> bytes:
+    data = np.array([[55]], dtype=np.uint8)
+    with MemoryFile() as memfile:
+        with memfile.open(
+            driver="GTiff",
+            height=data.shape[0],
+            width=data.shape[1],
+            count=1,
+            dtype=data.dtype,
+            crs="EPSG:27700",
+            transform=from_origin(0, 1, 1, 1),
+        ) as dataset:
+            dataset.write(data, 1)
+        return bytes(memfile.getbuffer())
+
+
 def test_download_tile_splits_after_retries(monkeypatch, tmp_path):
     monkeypatch.setattr(noise, "MAX_RETRIES", 1)
     monkeypatch.setattr(noise, "MIN_TILE_SIZE", 50)
+    tile_bytes = _tiny_geotiff_bytes()
 
     def fake_fetch_tile_bytes(
         wcs_base,
@@ -22,7 +40,7 @@ def test_download_tile_splits_after_retries(monkeypatch, tmp_path):
     ):
         if max_e - min_e > 50 or max_n - min_n > 50:
             raise httpx.TimeoutException("too large")
-        return b"II*\x00fake-tiff"
+        return tile_bytes
 
     monkeypatch.setattr(noise, "_fetch_tile_bytes", fake_fetch_tile_bytes)
 
@@ -107,6 +125,205 @@ def test_download_raster_raises_on_missing_strict_tiles(monkeypatch, tmp_path):
         noise.download_raster(tmp_path, "base", "coverage", "Road")
 
 
+def test_generate_tiles_neighbours_overlap_by_radius():
+    tile_size = 20_000
+    overlap = noise.POSTCODE_NOISE_RADIUS_M
+    tiles = noise._generate_tiles(
+        0, 60_000, 0, 60_000, tile_size, overlap, tile_size
+    )
+
+    by_origin = {(min_e, min_n): (max_e, max_n) for min_e, min_n, max_e, max_n in tiles}
+
+    # Horizontally adjacent tiles must overlap by >= overlap.
+    for (min_e, min_n), (max_e, _max_n) in by_origin.items():
+        right_origin = (min_e + tile_size, min_n)
+        if right_origin in by_origin:
+            assert max_e - right_origin[0] >= overlap
+
+    # Vertically adjacent tiles must overlap by >= overlap.
+    for (min_e, min_n), (_max_e, max_n) in by_origin.items():
+        up_origin = (min_e, min_n + tile_size)
+        if up_origin in by_origin:
+            assert max_n - up_origin[1] >= overlap
+
+
+def test_generate_tiles_clamps_to_grid_extent():
+    tile_size = 20_000
+    overlap = noise.POSTCODE_NOISE_RADIUS_M
+    tiles = noise._generate_tiles(
+        noise.BNG_MAX_E - tile_size,
+        noise.BNG_MAX_E,
+        noise.BNG_MAX_N - tile_size,
+        noise.BNG_MAX_N,
+        tile_size,
+        overlap,
+        tile_size,
+    )
+    # The final (top-right) tile cannot extend past the England extent even
+    # though the overlap would otherwise push it beyond.
+    for _min_e, _min_n, max_e, max_n in tiles:
+        assert max_e <= noise.BNG_MAX_E
+        assert max_n <= noise.BNG_MAX_N
+
+
+def _write_geotiff(path, data, left, top, resolution, nodata):
+    with rasterio.open(
+        path,
+        "w",
+        driver="GTiff",
+        height=data.shape[0],
+        width=data.shape[1],
+        count=1,
+        dtype=data.dtype,
+        crs="EPSG:27700",
+        transform=from_origin(left, top, resolution, resolution),
+        nodata=nodata,
+    ) as dataset:
+        dataset.write(data, 1)
+
+
+def test_sample_noise_recovers_value_across_overlapping_seam(monkeypatch, tmp_path):
+    monkeypatch.setattr(noise, "POSTCODE_NOISE_RADIUS_M", 50)
+    monkeypatch.setattr(noise, "RESOLUTION", 10)
+
+    # Two download tiles share a vertical seam at easting=100. _generate_tiles
+    # decides their real footprints: with the overlap fix the LEFT tile extends
+    # past the seam by POSTCODE_NOISE_RADIUS_M and thus covers a loud cell that
+    # physically sits just across the seam.
+    tile_size = 100
+    overlap = noise.POSTCODE_NOISE_RADIUS_M
+    tiles = noise._generate_tiles(0, 200, 0, 100, tile_size, overlap, tile_size)
+    by_origin = {
+        (min_e, min_n): (max_e, max_n) for min_e, min_n, max_e, max_n in tiles
+    }
+    left_min_e, left_min_n = 0, 0
+    left_max_e, left_max_n = by_origin[(left_min_e, left_min_n)]
+    # Overlap fix is what makes the left tile reach across the seam.
+    assert left_max_e > 100
+
+    # The loud 70 dB cell centre is at easting 105 (just across the seam) and
+    # the postcode point is at easting 75 in the left tile, within 50m of it.
+    res = noise.RESOLUTION
+    width = int((left_max_e - left_min_e) // res)
+    height = int((left_max_n - left_min_n) // res)
+    left_data = np.zeros((height, width), dtype=np.float32)
+    loud_row = height - 1 - int((25 - left_min_n) // res)  # northing ~25
+    loud_col = int((105 - left_min_e) // res)  # easting ~105
+    left_data[loud_row, loud_col] = 70.0
+    _write_geotiff(
+        tmp_path / "left.tif", left_data, left_min_e, left_max_n, res, nodata=0
+    )
+
+    # The right tile holds the same loud cell but the postcode point is NOT
+    # inside it, so without overlap the value would be lost for that point.
+    right_min_e, right_min_n = 100, 0
+    right_max_e, right_max_n = by_origin[(right_min_e, right_min_n)]
+    rwidth = int((right_max_e - right_min_e) // res)
+    rheight = int((right_max_n - right_min_n) // res)
+    right_data = np.zeros((rheight, rwidth), dtype=np.float32)
+    right_data[rheight - 1 - int((25 - right_min_n) // res), 0] = 70.0
+    _write_geotiff(
+        tmp_path / "right.tif", right_data, right_min_e, right_max_n, res, nodata=0
+    )
+
+    result = noise.sample_noise_at_postcodes(
+        [tmp_path / "left.tif", tmp_path / "right.tif"],
+        easting=np.array([75.0]),
+        northing=np.array([25.0]),
+        label="Road",
+        col_name="road_noise_lden_db",
+    )
+
+    assert result.to_list() == [70.0]
+
+
+def test_sample_noise_distinguishes_nodata_from_in_coverage_quiet(
+    monkeypatch, tmp_path
+):
+    monkeypatch.setattr(noise, "POSTCODE_NOISE_RADIUS_M", 0)
+    monkeypatch.setattr(noise, "RESOLUTION", 10)
+
+    # Defra encodes TRUE nodata as the -96.0 sentinel; genuinely quiet ground
+    # below the lowest reporting band is 0.0. With a 0m radius each postcode
+    # reads exactly one cell, so we can pin behaviour per cell:
+    #   -96.0 sentinel  -> null  ("we don't know")
+    #   0.0 in-coverage -> NOISE_QUIET_FLOOR_DB ("we know it's quiet")
+    #   65.0            -> 65.0  (a real modelled reading)
+    data = np.array(
+        [
+            [-96.0, 0.0, 65.0],
+        ],
+        dtype=np.float32,
+    )
+    _write_geotiff(tmp_path / "noise.tif", data, 0, 10, 10, nodata=-96.0)
+
+    result = noise.sample_noise_at_postcodes(
+        [tmp_path / "noise.tif"],
+        # Cell centres at easting 5 (nodata), 15 (quiet 0.0), 25 (loud 65).
+        easting=np.array([5.0, 15.0, 25.0]),
+        northing=np.array([5.0, 5.0, 5.0]),
+        label="Road",
+        col_name="road_noise_lden_db",
+    )
+
+    assert result.to_list() == [None, float(noise.NOISE_QUIET_FLOOR_DB), 65.0]
+
+
+def test_sample_noise_preserves_genuine_reading_above_quiet_floor(monkeypatch, tmp_path):
+    monkeypatch.setattr(noise, "POSTCODE_NOISE_RADIUS_M", 0)
+    monkeypatch.setattr(noise, "RESOLUTION", 10)
+
+    # The lowest Defra reporting band is 40.0 dB; genuine readings populate
+    # [40, ~80]. A genuine in-coverage reading at or just above the floor must be
+    # PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A
+    # quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
+    # dB readings; this pins that they survive unchanged.
+    floor = float(noise.NOISE_QUIET_FLOOR_DB)
+    data = np.array(
+        [
+            [42.0, floor, 0.0],
+        ],
+        dtype=np.float32,
+    )
+    _write_geotiff(tmp_path / "noise.tif", data, 0, 10, 10, nodata=-96.0)
+
+    result = noise.sample_noise_at_postcodes(
+        [tmp_path / "noise.tif"],
+        # Cell centres at easting 5 (42 dB), 15 (floor), 25 (quiet 0.0).
+        easting=np.array([5.0, 15.0, 25.0]),
+        northing=np.array([5.0, 5.0, 5.0]),
+        label="Road",
+        col_name="road_noise_lden_db",
+    )
+
+    # 42 preserved (NOT raised to the floor), floor preserved, 0.0 -> floor.
+    assert result.to_list() == [42.0, floor, floor]
+    # The floor must sit at/below the lowest genuine reading so nothing inflates.
+    assert floor <= 42.0
+
+
+def test_sample_noise_nodata_window_stays_null(monkeypatch, tmp_path):
+    monkeypatch.setattr(noise, "POSTCODE_NOISE_RADIUS_M", 15)
+    monkeypatch.setattr(noise, "RESOLUTION", 10)
+
+    # A postcode whose entire 3x3 max-window is the -96.0 sentinel must remain
+    # null: no in-coverage cell was read, so "quiet" must NOT be inferred.
+    data = np.full((5, 5), -96.0, dtype=np.float32)
+    data[4, 4] = 70.0  # one loud cell, far from the nodata corner
+    _write_geotiff(tmp_path / "noise.tif", data, 0, 50, 10, nodata=-96.0)
+
+    result = noise.sample_noise_at_postcodes(
+        [tmp_path / "noise.tif"],
+        # Top-left point: its 3x3 window is cells (rows 0-1, cols 0-1) = all -96.
+        easting=np.array([5.0]),
+        northing=np.array([45.0]),
+        label="Road",
+        col_name="road_noise_lden_db",
+    )
+
+    assert result.to_list() == [None]
+
+
 def test_sample_noise_at_postcodes_uses_local_maximum(monkeypatch, tmp_path):
     monkeypatch.setattr(noise, "POSTCODE_NOISE_RADIUS_M", 15)
     monkeypatch.setattr(noise, "RESOLUTION", 10)

pipeline/download/test_places.py

@@ -1,15 +1,24 @@
+import numpy as np
 import polars as pl
 from pyproj import Transformer
+from scipy.spatial import cKDTree
 
 from pipeline.download.places import (
     _assign_london_display_city,
+    _build_street_places,
     _display_city_from_tags,
     _is_dlr_station,
     _is_tram_station,
+    _london_postcode_tree,
     _naptan_dlr_stations,
+    _normalize_street_name,
     _ofs_universities,
+    _outcode_of_postcode,
+    _outcode_tree,
+    _pois_to_places,
     _select_university_name,
     _station_display_name,
+    _street_centroid,
 )
 
 WGS84_TO_BNG = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
@@ -168,6 +177,109 @@ def test_ofs_universities_extracts_university_title_rows_with_postcode_coords():
     ]
 
 
+def test_street_centroid_averages_vertices():
+    assert _street_centroid([(51.0, -0.1), (53.0, -0.3)]) == (52.0, -0.2)
+    assert _street_centroid([]) is None
+
+
+def test_normalize_street_name_and_outcode():
+    assert _normalize_street_name("  High   Street ") == "high street"
+    assert _outcode_of_postcode("NW1 6XE") == "NW1"
+    assert _outcode_of_postcode("") == ""
+
+
+def test_build_street_places_groups_segments_by_name_and_outcode():
+    # Two postcodes: NW1 (north) and CR0 (south).
+    tree = cKDTree(np.array([[51.53, -0.14], [51.37, -0.10]], dtype=np.float64))
+    outcodes = ["NW1", "CR0"]
+
+    streets = [
+        {"name": "High Street", "lat": 51.531, "lon": -0.141},  # NW1
+        {"name": "High Street", "lat": 51.529, "lon": -0.139},  # NW1 (same road, 2nd segment)
+        {"name": "High Street", "lat": 51.371, "lon": -0.101},  # CR0 (different road, same name)
+        {"name": "Baker Street", "lat": 51.5305, "lon": -0.1405},  # NW1
+    ]
+
+    places = _build_street_places(streets, tree, outcodes)
+
+    # 3 distinct streets: High Street/NW1 (2 segments merged), High Street/CR0, Baker Street/NW1.
+    assert len(places) == 3
+    assert all(place["place_type"] == "street" for place in places)
+
+    nw1_high = next(
+        place
+        for place in places
+        if place["name"] == "High Street" and place["lat"] > 51.5
+    )
+    assert nw1_high["lat"] == (51.531 + 51.529) / 2
+    assert nw1_high["lon"] == (-0.141 + -0.139) / 2
+    # The same-named CR0 road stays separate.
+    assert any(
+        place["name"] == "High Street" and place["lat"] < 51.4 for place in places
+    )
+
+
+def test_pois_to_places_keeps_high_value_named_pois_only():
+    pois = pl.DataFrame(
+        {
+            "name": ["Hyde Park", "St Thomas' Hospital", "Joe's Cafe", "Hyde Park", ""],
+            "category": [
+                "leisure/park",
+                "amenity/hospital",
+                "amenity/cafe",  # everyday amenity → excluded
+                "leisure/park",  # duplicate → excluded
+                "leisure/park",  # empty name → excluded
+            ],
+            "lat": [51.507, 51.498, 51.5, 51.507, 51.6],
+            "lng": [-0.165, -0.118, -0.1, -0.165, -0.2],
+        }
+    )
+
+    places = _pois_to_places(pois)
+
+    assert [(place["name"], place["place_type"]) for place in places] == [
+        ("Hyde Park", "park"),
+        ("St Thomas' Hospital", "hospital"),
+    ]
+    assert all(place["travel_destination"] is False for place in places)
+
+
+def test_pois_to_places_keeps_distinct_same_named_pois():
+    # Two genuinely distinct POIs sharing a name, far apart (London vs Bristol).
+    pois = pl.DataFrame(
+        {
+            "name": ["Victoria Park", "Victoria Park"],
+            "category": ["leisure/park", "leisure/park"],
+            "lat": [51.54, 51.46],
+            "lng": [-0.04, -2.60],
+        }
+    )
+
+    places = _pois_to_places(pois)
+
+    assert len(places) == 2
+    assert {(place["lat"], place["lon"]) for place in places} == {
+        (51.54, -0.04),
+        (51.46, -2.60),
+    }
+
+
+def test_pois_to_places_still_dedupes_colocated():
+    # The same physical POI mapped twice a few metres apart collapses to one.
+    pois = pl.DataFrame(
+        {
+            "name": ["Victoria Park", "Victoria Park"],
+            "category": ["leisure/park", "leisure/park"],
+            "lat": [51.5400, 51.5401],
+            "lng": [-0.0400, -0.0399],
+        }
+    )
+
+    places = _pois_to_places(pois)
+
+    assert len(places) == 1
+
+
 def test_display_city_from_tags_uses_explicit_london_context():
     assert _display_city_from_tags({"is_in": "Croydon, London, UK"}) == "London"
     assert _display_city_from_tags({"is_in": "Croydon, Cambridgeshire, UK"}) is None
@@ -216,3 +328,52 @@ def test_assign_london_display_city_uses_nearest_active_postcode_admin(tmp_path)
 
     assert assigned == 2
     assert [place["display_city"] for place in places] == ["London", "London", None]
+
+
+def test_no_grid_reference_sentinel_is_excluded_from_coordinate_trees(tmp_path):
+    # ONS NSPL stores postcodes with no grid reference at the Null-Island sentinel
+    # lat=99.999999, long=0.0, whose paired BNG coords collapse to the (0, 0) origin.
+    # Such an active postcode must never enter the nearest-neighbour indexes.
+    sentinel = {
+        "pcds": "ZZ99 9ZZ",
+        "lat": 99.999999,
+        "long": 0.0,
+        "doterm": None,
+        "ctry25cd": "E92000001",
+        "east1m": 0,
+        "north1m": 0,
+        "rgn25cd": "E12000007",
+        "lad25cd": "E09000008",
+        "cty25cd": "E13000002",
+    }
+    croydon_easting, croydon_northing = WGS84_TO_BNG.transform(-0.101793, 51.371273)
+    real = {
+        "pcds": "CR0 1SZ",
+        "lat": 51.371273,
+        "long": -0.101793,
+        "doterm": None,
+        "ctry25cd": "E92000001",
+        "east1m": int(round(croydon_easting)),
+        "north1m": int(round(croydon_northing)),
+        "rgn25cd": "E12000007",
+        "lad25cd": "E09000008",
+        "cty25cd": "E13000002",
+    }
+    postcodes = tmp_path / "postcodes.parquet"
+    pl.DataFrame([sentinel, real]).write_parquet(postcodes)
+
+    # lat/long outcode tree: only the real postcode survives, so a London-area query
+    # cannot be tagged with the sentinel's (empty) outcode.
+    tree, outcodes = _outcode_tree(postcodes)
+    assert tree.n == 1
+    assert outcodes == ["CR0"]
+    _, idx = tree.query([[51.371273, -0.101793]])
+    assert outcodes[idx[0]] == "CR0"
+
+    # BNG London tree: only the real postcode survives, so the (0, 0) origin can never
+    # be the nearest neighbour of a real place.
+    bng_tree, london_flags = _london_postcode_tree(postcodes)
+    assert bng_tree.n == 1
+    assert london_flags.tolist() == [True]
+    _, bng_idx = bng_tree.query([[croydon_easting, croydon_northing]])
+    assert bng_idx[0] == 0

pipeline/download/test_satellite_highres.py

@@ -0,0 +1,97 @@
+from pipeline.download import satellite_highres
+from pipeline.download.satellite_highres import (
+    VapTile,
+    parse_search_results,
+    select_best_rgb_tiles,
+)
+
+
+def _result(product: str, year: str, resolution: str, tile: str) -> dict:
+    """One search-API record in the real response shape."""
+    return {
+        "product": {"id": product, "label": product},
+        "year": {"id": year, "label": year},
+        "resolution": {"id": resolution, "label": f"{resolution}m"},
+        "tile": {"id": tile, "label": tile},
+        "label": f"{product}-{year}-{resolution}m-{tile}",
+        "uri": (
+            "https://environment.data.gov.uk/tiles/collections/survey/"
+            f"{product}/{year}/{resolution}/{tile}"
+        ),
+    }
+
+
+# Mirrors a real Greater-London response: RGB at 0.4m (2008) and 0.1m (2011),
+# plus Night Time and LIDAR products that must be ignored.
+SAMPLE_PAYLOAD = {
+    "count": 6,
+    "results": [
+        _result("vertical_aerial_photography_tiles_rgb", "2008", "0.4", "TQ2575"),
+        _result("vertical_aerial_photography_tiles_night_time", "2012", "0.2", "TQ2575"),
+        _result("lidar_composite_dtm", "2022", "1", "TQ2575"),
+        # TQ3080 has two RGB captures: a finer-but-older and a coarser-but-newer.
+        _result("vertical_aerial_photography_tiles_rgb", "2008", "0.1", "TQ3080"),
+        _result("vertical_aerial_photography_tiles_rgb", "2011", "0.25", "TQ3080"),
+        _result("vertical_aerial_photography_tiles_irrgb", "2012", "0.5", "TQ3080"),
+    ],
+}
+
+
+def test_parse_search_results_skips_malformed_records() -> None:
+    payload = {
+        "results": [
+            _result("vertical_aerial_photography_tiles_rgb", "2008", "0.4", "TQ2575"),
+            {"product": {"id": "broken"}},  # missing year/resolution/tile/uri
+        ]
+    }
+    tiles = parse_search_results(payload)
+    assert len(tiles) == 1
+    assert tiles[0] == VapTile(
+        product_id="vertical_aerial_photography_tiles_rgb",
+        year=2008,
+        resolution_m=0.4,
+        os_tile_id="TQ2575",
+        uri="https://environment.data.gov.uk/tiles/collections/survey/"
+        "vertical_aerial_photography_tiles_rgb/2008/0.4/TQ2575",
+        label="vertical_aerial_photography_tiles_rgb-2008-0.4m-TQ2575",
+    )
+
+
+def test_select_best_rgb_filters_non_rgb_products() -> None:
+    selected = select_best_rgb_tiles(parse_search_results(SAMPLE_PAYLOAD))
+    assert {tile.product_id for tile in selected} == {
+        satellite_highres.VAP_RGB_PRODUCT
+    }
+
+
+def test_select_best_rgb_one_tile_per_os_square() -> None:
+    selected = select_best_rgb_tiles(parse_search_results(SAMPLE_PAYLOAD))
+    assert sorted(tile.os_tile_id for tile in selected) == ["TQ2575", "TQ3080"]
+
+
+def test_select_best_rgb_prefers_finest_resolution_then_latest_year() -> None:
+    selected = {
+        tile.os_tile_id: tile
+        for tile in select_best_rgb_tiles(parse_search_results(SAMPLE_PAYLOAD))
+    }
+    # TQ2575: only one RGB capture.
+    assert selected["TQ2575"].resolution_m == 0.4
+    # TQ3080: finest resolution (0.1m) wins even though it is the older survey.
+    assert selected["TQ3080"].resolution_m == 0.1
+    assert selected["TQ3080"].year == 2008
+
+
+def test_select_best_rgb_breaks_resolution_ties_by_year() -> None:
+    tiles = [
+        VapTile(satellite_highres.VAP_RGB_PRODUCT, 2009, 0.25, "TQ0101", "u", "a"),
+        VapTile(satellite_highres.VAP_RGB_PRODUCT, 2018, 0.25, "TQ0101", "u", "b"),
+        VapTile(satellite_highres.VAP_RGB_PRODUCT, 2015, 0.25, "TQ0101", "u", "c"),
+    ]
+    selected = select_best_rgb_tiles(tiles)
+    assert len(selected) == 1
+    assert selected[0].year == 2018
+
+
+def test_select_best_rgb_empty_when_no_rgb() -> None:
+    payload = {"results": [_result("lidar_composite_dtm", "2022", "1", "TQ2575")]}
+    assert select_best_rgb_tiles(parse_search_results(payload)) == []

pipeline/download/test_transit_network.py

@@ -0,0 +1,79 @@
+"""Tests for transit_network GTFS processing."""
+
+import zipfile
+from pathlib import Path
+
+import pytest
+
+from pipeline.download.transit_network import convert_high_freq_to_frequency_based
+
+
+def _write_gtfs(path: Path, *, stop_times: str) -> None:
+    """Write a minimal GTFS zip with one metro route and several trips."""
+    routes = "route_id,route_type\nR1,1\n"
+    trips = "trip_id,route_id,direction_id,service_id\n" + "".join(
+        f"T{i},R1,0,S1\n" for i in range(1, 7)
+    )
+    with zipfile.ZipFile(path, "w") as z:
+        z.writestr("routes.txt", routes)
+        z.writestr("trips.txt", trips)
+        z.writestr("stop_times.txt", stop_times)
+
+
+def _one_based_stop_times() -> str:
+    """Six trips, 1-based stop_sequence (1,2,...), 5-minute headway."""
+    header = "trip_id,stop_sequence,departure_time,stop_id\n"
+    rows = []
+    # First departures 06:00, 06:05, ... (300s = 5 min headway, well under 15 min)
+    for i in range(6):
+        trip = f"T{i + 1}"
+        first_dep = 6 * 3600 + i * 300
+        h, m = divmod(first_dep, 3600)
+        m, s = divmod(m, 60)
+        # First stop has stop_sequence 1 (NOT 0); second stop sequence 2.
+        rows.append(f"{trip},1,{h:02d}:{m:02d}:{s:02d},STOP_A\n")
+        later = first_dep + 120
+        h2, m2 = divmod(later, 3600)
+        m2, s2 = divmod(m2, 60)
+        rows.append(f"{trip},2,{h2:02d}:{m2:02d}:{s2:02d},STOP_B\n")
+    return header + "".join(rows)
+
+
+def test_one_based_stop_sequence_is_converted(tmp_path: Path) -> None:
+    """First stop selection must use the minimum stop_sequence, not literal "0".
+
+    With 1-based stop_sequence the old code (keyed on stop_sequence == "0") found
+    zero first stops and produced an empty frequencies.txt. The fix selects the
+    minimum stop_sequence per trip, so the high-frequency group is converted.
+    """
+    src = tmp_path / "in.zip"
+    dst = tmp_path / "out.zip"
+    _write_gtfs(src, stop_times=_one_based_stop_times())
+
+    convert_high_freq_to_frequency_based(src, dst)
+
+    with zipfile.ZipFile(dst, "r") as z:
+        freq = z.read("frequencies.txt").decode("utf-8")
+
+    freq_rows = [r for r in freq.splitlines()[1:] if r.strip()]
+    # The single high-frequency group must produce exactly one frequency entry.
+    assert len(freq_rows) == 1, freq
+    trip_id, start_time, end_time, headway_secs, _exact = freq_rows[0].split(",")
+    # Template trip is the earliest departure (T1 at 06:00) starting at first stop.
+    assert start_time == "06:00:00"
+    # Median headway of 300s rounds to a 300s headway entry.
+    assert headway_secs == "300"
+
+
+def test_raises_when_no_first_stops_found(tmp_path: Path) -> None:
+    """A non-empty target trip set with unparseable stop_sequence is loud, not silent."""
+    src = tmp_path / "in.zip"
+    dst = tmp_path / "out.zip"
+    bad = (
+        "trip_id,stop_sequence,departure_time,stop_id\n"
+        "T1,not_a_number,06:00:00,STOP_A\n"
+    )
+    _write_gtfs(src, stop_times=bad)
+
+    with pytest.raises(RuntimeError, match="no first stops"):
+        convert_high_freq_to_frequency_based(src, dst)

pipeline/download/transit_network.py

@@ -307,9 +307,14 @@ def convert_high_freq_to_frequency_based(
 
         print(f"  Found {len(trip_group_key)} trips on target routes")
 
-        # Step 3: Get first departure time and first stop for each target trip
+        # Step 3: Get first departure time and first stop for each target trip.
+        # GTFS only requires stop_sequence to be strictly increasing per trip; it
+        # is NOT required to start at 0 (1-based numbering is common, and BODS is
+        # consumed raw here without renumbering). So pick the row with the minimum
+        # stop_sequence per trip rather than keying off the literal "0".
         trip_first_dep: dict[str, int] = {}
         trip_first_stop: dict[str, str] = {}
+        trip_min_seq: dict[str, int] = {}
         with zin.open("stop_times.txt") as f:
             cols = _parse_csv_line(f.readline())
             trip_id_idx = cols.index("trip_id")
@@ -323,12 +328,26 @@ def convert_high_freq_to_frequency_based(
                 trip_id = parts[trip_id_idx].strip('"')
                 if trip_id not in trip_group_key:
                     continue
-                if parts[seq_idx].strip('"') == "0":
+                try:
+                    seq = int(parts[seq_idx].strip('"'))
+                except ValueError:
+                    continue
+                if trip_id in trip_min_seq and seq >= trip_min_seq[trip_id]:
+                    continue
                 dep_secs = _parse_gtfs_time(parts[dep_idx])
-                    if dep_secs is not None:
+                if dep_secs is None:
+                    continue
+                trip_min_seq[trip_id] = seq
                 trip_first_dep[trip_id] = dep_secs
                 trip_first_stop[trip_id] = parts[stop_id_idx].strip('"')
 
+    if trip_group_key and not trip_first_dep:
+        raise RuntimeError(
+            "convert_high_freq_to_frequency_based found no first stops for "
+            f"{len(trip_group_key)} target trips; stop_times.txt may be malformed "
+            "or stop_sequence parsing failed"
+        )
+
     # Step 4: Group trips by (route, direction, service, first_stop) and compute headways
     groups: dict[tuple[str, ...], list[tuple[str, int]]] = defaultdict(list)
     for trip_id, dep_secs in trip_first_dep.items():

pipeline/download/uprn_lookup.py

@@ -14,7 +14,7 @@ from pathlib import Path
 import polars as pl
 
 from pipeline.local_temp import local_tmp_dir
-from pipeline.utils import download, extract_zip
+from pipeline.utils import code_col_overrides, download, extract_zip
 
 URL = "https://www.arcgis.com/sharing/rest/content/items/4e0b4b3fbc2540caae27e7be532e61be/data"
 
@@ -34,16 +34,16 @@ def find_csvs(extract_path: Path) -> list[Path]:
 
 def convert_to_parquet(csv_paths: list[Path], parquet_path: Path) -> None:
     # Some regional files infer different types for the same column (e.g.
-    # ruc21ind is String in most but Int64 in YH). Read all code columns as
-    # String to avoid schema mismatches.
-    CODE_COLS = {
-        "ruc21ind": pl.String,
-        "oac21ind": pl.String,
-        "imd19ind": pl.String,
-    }
+    # ruc21ind is String in most but Int64 in YH), and string codes like "UN1"
+    # appear deep in the data. Read all classification-index code columns as
+    # String to avoid schema mismatches. NSUL renames the year suffixes each
+    # release and polars silently ignores overrides for missing columns, so
+    # match on the suffix-free stem (from the header) rather than hard-coding.
+    names = pl.scan_csv(csv_paths[0]).collect_schema().names()
+    code_cols = code_col_overrides(names)
     df = pl.concat(
         [
-            pl.scan_csv(p, try_parse_dates=True, schema_overrides=CODE_COLS)
+            pl.scan_csv(p, try_parse_dates=True, schema_overrides=code_cols)
             for p in csv_paths
         ]
     )

pipeline/test_validate_outputs.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import zipfile
 
 import polars as pl
@@ -7,6 +8,32 @@ import polars as pl
 from pipeline.validate_outputs import main
 
 
+def polygon(offset=0):
+    x = float(offset)
+    return {
+        "type": "Polygon",
+        "coordinates": [
+            [(x, 0.0), (x + 0.001, 0.0), (x + 0.001, 0.001), (x, 0.001), (x, 0.0)]
+        ],
+    }
+
+
+def write_boundary(path, postcodes, geometries=None):
+    units = path / "units"
+    units.mkdir(parents=True)
+    features = [
+        {
+            "type": "Feature",
+            "properties": {"postcodes": postcode},
+            "geometry": (geometries[index] if geometries else polygon(index)),
+        }
+        for index, postcode in enumerate(postcodes)
+    ]
+    (units / "AA1.geojson").write_text(
+        json.dumps({"type": "FeatureCollection", "features": features})
+    )
+
+
 def test_validates_parquet_file_and_zip(tmp_path, monkeypatch):
     parquet_path = tmp_path / "data.parquet"
     file_path = tmp_path / "plain.txt"
@@ -59,3 +86,310 @@ def test_rejects_missing_and_empty_outputs(tmp_path, monkeypatch, capsys):
     assert "empty file" in stderr
     assert "missing" in stderr
     assert "no files matched" in stderr
+
+
+def test_validates_postcode_boundary_matches(tmp_path, monkeypatch):
+    postcodes_path = tmp_path / "postcodes.parquet"
+    boundaries_path = tmp_path / "postcode_boundaries"
+    pl.DataFrame({"postcode": ["AA1 1AA", "AA1 1AB"]}).write_parquet(postcodes_path)
+    write_boundary(boundaries_path, ["AA1 1AA", "AA1 1AB"])
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate_outputs",
+            "--postcode-boundary-match",
+            f"{postcodes_path}::{boundaries_path}",
+        ],
+    )
+
+    assert main() == 0
+
+
+def test_rejects_postcode_boundary_mismatch(tmp_path, monkeypatch, capsys):
+    postcodes_path = tmp_path / "postcodes.parquet"
+    boundaries_path = tmp_path / "postcode_boundaries"
+    pl.DataFrame({"postcode": ["AA1 1AA", "AA1 1AB"]}).write_parquet(postcodes_path)
+    write_boundary(boundaries_path, ["AA1 1AA", "AA1 1AC"])
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate_outputs",
+            "--postcode-boundary-match",
+            f"{postcodes_path}::{boundaries_path}",
+        ],
+    )
+
+    assert main() == 1
+    stderr = capsys.readouterr().err
+    assert "missing boundaries" in stderr
+    assert "boundary postcodes are absent" in stderr
+
+
+def test_rejects_invalid_postcode_boundary_features(tmp_path, monkeypatch, capsys):
+    postcodes_path = tmp_path / "postcodes.parquet"
+    boundaries_path = tmp_path / "postcode_boundaries"
+    units = boundaries_path / "units"
+    units.mkdir(parents=True)
+    pl.DataFrame({"postcode": ["AA1 1AA"]}).write_parquet(postcodes_path)
+    bowtie = {
+        "type": "Polygon",
+        "coordinates": [[(0, 0), (1, 1), (1, 0), (0, 1), (0, 0)]],
+    }
+    features = [
+        {
+            "type": "Feature",
+            "properties": {"postcodes": "AA1 1AA"},
+            "geometry": polygon(),
+        },
+        {
+            "type": "Feature",
+            "properties": {"postcodes": "AA1 1AA"},
+            "geometry": polygon(1),
+        },
+        {"type": "Feature", "properties": {}, "geometry": polygon(2)},
+        {"type": "Feature", "properties": {"postcodes": "AA1 1AB"}, "geometry": None},
+        {"type": "Feature", "properties": {"postcodes": "AA1 1AC"}, "geometry": bowtie},
+    ]
+    (units / "AA1.geojson").write_text(
+        json.dumps({"type": "FeatureCollection", "features": features})
+    )
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate_outputs",
+            "--postcode-boundary-match",
+            f"{postcodes_path}::{boundaries_path}",
+        ],
+    )
+
+    assert main() == 1
+    stderr = capsys.readouterr().err
+    assert "duplicate boundary postcode features" in stderr
+    assert "missing properties.postcodes" in stderr
+    assert "missing or empty geometry" in stderr
+    assert "invalid boundary geometries" in stderr
+
+
+def test_validates_active_english_arcgis_boundary_matches(tmp_path, monkeypatch):
+    arcgis_path = tmp_path / "arcgis.parquet"
+    boundaries_path = tmp_path / "postcode_boundaries"
+    pl.DataFrame(
+        {
+            "pcds": ["AA1 1AA", "AA1 1AB", "CF1 1AA"],
+            "ctry25cd": ["E92000001", "E92000001", "W92000004"],
+            "doterm": [None, "2020-01-01", None],
+        }
+    ).write_parquet(arcgis_path)
+    write_boundary(boundaries_path, ["AA1 1AA"])
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate_outputs",
+            "--active-postcode-boundary-match",
+            f"{arcgis_path}::{boundaries_path}",
+        ],
+    )
+
+    assert main() == 0
+
+
+def test_rejects_active_english_arcgis_boundary_mismatch(tmp_path, monkeypatch, capsys):
+    arcgis_path = tmp_path / "arcgis.parquet"
+    boundaries_path = tmp_path / "postcode_boundaries"
+    pl.DataFrame(
+        {
+            "pcds": ["AA1 1AA", "AA1 1AB", "CF1 1AA"],
+            "ctry25cd": ["E92000001", "E92000001", "W92000004"],
+            "doterm": [None, None, None],
+        }
+    ).write_parquet(arcgis_path)
+    write_boundary(boundaries_path, ["AA1 1AA", "CF1 1AA"])
+
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate_outputs",
+            "--active-postcode-boundary-match",
+            f"{arcgis_path}::{boundaries_path}",
+        ],
+    )
+
+    assert main() == 1
+    stderr = capsys.readouterr().err
+    assert "active English postcodes" in stderr
+    assert "not active English postcodes" in stderr
+
+
+def _write_postcode_features(path, rows):
+    pl.DataFrame(rows).write_parquet(path)
+
+
+def test_validates_postcode_features_valid(tmp_path, monkeypatch):
+    path = tmp_path / "postcode.parquet"
+    _write_postcode_features(
+        path,
+        {
+            "Postcode": ["AA1 1AA", "BB1 1BB"],
+            "lat": [51.5, 53.4],
+            "lon": [-0.1, -2.2],
+            "ctry25cd": ["E92000001", "E92000001"],
+            "% White": [80.0, 55.0],
+        },
+    )
+    monkeypatch.setattr("sys.argv", ["validate", "--postcode-features", str(path)])
+    assert main() == 0
+
+
+def test_rejects_contaminated_postcode_features(tmp_path, monkeypatch, capsys):
+    path = tmp_path / "postcode.parquet"
+    _write_postcode_features(
+        path,
+        {
+            "Postcode": ["AA1 1AA", "AA1 1AA", "CF10 1AA"],  # duplicate AA1 1AA
+            "lat": [51.5, 51.5, None],  # Welsh row has null coord
+            "lon": [-0.1, -0.1, None],
+            "ctry25cd": ["E92000001", "E92000001", "W92000004"],
+            "% White": [80.0, 150.0, 90.0],  # 150 out of [0,100]
+        },
+    )
+    monkeypatch.setattr("sys.argv", ["validate", "--postcode-features", str(path)])
+    assert main() == 1
+    err = capsys.readouterr().err
+    assert "not unique" in err
+    assert "E92000001" in err  # country contamination
+    assert "out-of-England" in err or "lat/lon" in err
+    assert "[0, 100]" in err
+
+
+def test_postcode_universe_rejects_missing(tmp_path, monkeypatch, capsys):
+    arcgis_path = tmp_path / "arcgis.parquet"
+    postcodes_path = tmp_path / "postcode.parquet"
+    pl.DataFrame(
+        {
+            "pcds": ["AA1 1AA", "AA1 1AB", "AA1 1AC"],
+            "ctry25cd": ["E92000001", "E92000001", "E92000001"],
+            "doterm": [None, None, None],
+        }
+    ).write_parquet(arcgis_path)
+    # Only 1 of the 3 active English postcodes is present, all otherwise valid.
+    _write_postcode_features(
+        postcodes_path,
+        {
+            "Postcode": ["AA1 1AA"],
+            "lat": [51.5],
+            "lon": [-0.1],
+            "ctry25cd": ["E92000001"],
+            "% White": [80.0],
+        },
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate",
+            "--postcode-universe",
+            f"{arcgis_path}::{postcodes_path}",
+        ],
+    )
+    assert main() == 1
+    err = capsys.readouterr().err
+    assert "missing" in err
+    assert "2" in err  # 2 of the 3 active postcodes are absent
+
+
+def test_postcode_universe_accepts_exact_match(tmp_path, monkeypatch):
+    arcgis_path = tmp_path / "arcgis.parquet"
+    postcodes_path = tmp_path / "postcode.parquet"
+    pl.DataFrame(
+        {
+            "pcds": ["AA1 1AA", "AA1 1AB"],
+            "ctry25cd": ["E92000001", "E92000001"],
+            "doterm": [None, None],
+        }
+    ).write_parquet(arcgis_path)
+    _write_postcode_features(
+        postcodes_path,
+        {
+            "Postcode": ["AA1 1AA", "AA1 1AB"],
+            "lat": [51.5, 53.4],
+            "lon": [-0.1, -2.2],
+            "ctry25cd": ["E92000001", "E92000001"],
+            "% White": [80.0, 55.0],
+        },
+    )
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "validate",
+            "--postcode-universe",
+            f"{arcgis_path}::{postcodes_path}",
+        ],
+    )
+    assert main() == 0
+
+
+def test_validates_properties_subset(tmp_path, monkeypatch):
+    postcode = tmp_path / "postcode.parquet"
+    properties = tmp_path / "properties.parquet"
+    pl.DataFrame({"Postcode": ["AA1 1AA", "BB1 1BB"]}).write_parquet(postcode)
+    pl.DataFrame(
+        {"Postcode": ["AA1 1AA"], "Last known price": [250_000]}
+    ).write_parquet(properties)
+    monkeypatch.setattr(
+        "sys.argv",
+        ["validate", "--properties-subset", f"{properties}::{postcode}"],
+    )
+    assert main() == 0
+
+
+def test_rejects_orphan_properties(tmp_path, monkeypatch, capsys):
+    postcode = tmp_path / "postcode.parquet"
+    properties = tmp_path / "properties.parquet"
+    pl.DataFrame({"Postcode": ["AA1 1AA"]}).write_parquet(postcode)
+    pl.DataFrame(
+        {"Postcode": ["CC1 1CC"], "Last known price": [-5]}  # orphan + negative price
+    ).write_parquet(properties)
+    monkeypatch.setattr(
+        "sys.argv",
+        ["validate", "--properties-subset", f"{properties}::{postcode}"],
+    )
+    assert main() == 1
+    err = capsys.readouterr().err
+    assert "absent from" in err
+    assert "non-positive" in err
+
+
+def test_validates_price_index_allows_zero_n_pairs(tmp_path, monkeypatch):
+    path = tmp_path / "price_index.parquet"
+    pl.DataFrame(
+        {
+            "sector": ["A1 1", "A1 1", "B2 2"],
+            "type_group": ["All", "Detached", "All"],
+            "year": [2024, 2024, 2024],
+            "log_index": [0.5, 0.4, 0.0],
+            "n_pairs": [100, 0, 0],  # zero n_pairs is a legitimate fallback
+        }
+    ).write_parquet(path)
+    monkeypatch.setattr("sys.argv", ["validate", "--price-index", str(path)])
+    assert main() == 0
+
+
+def test_rejects_price_index_nonfinite_and_duplicate(tmp_path, monkeypatch, capsys):
+    path = tmp_path / "price_index.parquet"
+    pl.DataFrame(
+        {
+            "sector": ["A1 1", "A1 1"],
+            "type_group": ["All", "All"],  # duplicate (sector, type_group, year)
+            "year": [2024, 2024],
+            "log_index": [float("inf"), 0.3],  # non-finite
+            "n_pairs": [10, 10],
+        }
+    ).write_parquet(path)
+    monkeypatch.setattr("sys.argv", ["validate", "--price-index", str(path)])
+    assert main() == 1
+    err = capsys.readouterr().err
+    assert "non-finite" in err
+    assert "not unique" in err

pipeline/transform/crime.py

@@ -28,6 +28,17 @@ MINOR_CRIME_TYPES = (
     "Other crime",
 )
 
+# Legacy police.uk crime-type names (pre-2014 taxonomy) mapped to their closest
+# current equivalent. Without this, ~1.9M incidents from 2010-2013 ("Violent
+# crime", "Public disorder and weapons") are unrecognised and silently dropped,
+# which understates pre-2013 serious crime and creates an artificial 2012->2013
+# step in the by-year series. Applied with `.replace` (not `.replace_strict`) so
+# unmapped current types pass through unchanged.
+LEGACY_CRIME_TYPE_ALIASES = {
+    "Violent crime": "Violence and sexual offences",
+    "Public disorder and weapons": "Public order",
+}
+
 
 def find_street_crime_csvs(crime_dir: Path) -> tuple[list[Path], int]:
     csvs = sorted(crime_dir.rglob("*.csv"))
@@ -84,11 +95,14 @@ def transform_crime(
         f"({valid_months[0]} to {valid_months[-1]})"
     )
 
-    # Count monthly incidents, then annualise over every valid month in the dataset.
-    # `_weight` (≤1) comes from the LSOA 2011→2021 lookup: 2011 LSOAs that split
-    # into N 2021 LSOAs contribute 1/N of their count to each child, since we
-    # don't know which child a given incident actually belonged to.
-    yearly_counts = (
+    # Annualise each year separately (count_in_year * 12 / months_in_year), then
+    # take the simple mean of those per-year rates over the years each type is
+    # present. This makes the headline equal the average of the by-year chart bars
+    # (_write_crime_by_year) instead of a month-weighted pooled rate, mirroring
+    # crime_spatial._write_avg_yr. `_weight` (≤1) comes from the LSOA 2011→2021
+    # lookup: 2011 LSOAs that split into N 2021 LSOAs contribute 1/N of their count
+    # to each child, since we don't know which child an incident actually belonged to.
+    filtered = (
         df.filter(
             valid_month_expr
             & pl.col("LSOA code").is_not_null()
@@ -96,14 +110,31 @@ def transform_crime(
             & pl.col("Crime type").is_not_null()
             & (pl.col("Crime type") != "")
         )
-        .group_by("LSOA code", "Month", "Crime type")
-        .agg((pl.col("_weight").first() * pl.len()).alias("count"))
-        .group_by("LSOA code", "Crime type")
-        .agg(
-            (pl.col("count").sum() / pl.lit(valid_month_count) * 12)
-            .round(1)
-            .alias("yearly_avg")
+        .with_columns(
+            pl.col("Month").str.slice(0, 4).cast(pl.Int32).alias("year"),
+            pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES),
         )
+    )
+
+    # Months observed *anywhere* in the dataset for each year (annualisation
+    # denominator), matching the by-year output's per-year scaling.
+    months_per_year = filtered.group_by("year").agg(
+        pl.col("Month").n_unique().alias("months_in_year")
+    )
+
+    yearly_counts = (
+        filtered.group_by("LSOA code", "year", "Crime type", "Month")
+        .agg((pl.col("_weight").first() * pl.len()).alias("count"))
+        .group_by("LSOA code", "year", "Crime type")
+        .agg(pl.col("count").sum().alias("count"))
+        .join(months_per_year, on="year")
+        .with_columns(
+            (pl.col("count") * 12.0 / pl.col("months_in_year")).alias("per_year")
+        )
+        # Mean of the per-year annualised rates over the years the type is present
+        # (only years with rows are grouped here, so this is the correct x-span).
+        .group_by("LSOA code", "Crime type")
+        .agg(pl.col("per_year").mean().round(1).alias("yearly_avg"))
         .collect(engine="streaming")
     )
     if yearly_counts.is_empty():
@@ -147,7 +178,10 @@ def _write_crime_by_year(
         & (pl.col("LSOA code") != "")
         & pl.col("Crime type").is_not_null()
         & (pl.col("Crime type") != "")
-    ).with_columns(pl.col("Month").str.slice(0, 4).cast(pl.Int32).alias("year"))
+    ).with_columns(
+        pl.col("Month").str.slice(0, 4).cast(pl.Int32).alias("year"),
+        pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES),
+    )
 
     # Months observed *anywhere* in the dataset for each year (annualisation denominator).
     # Using crime-type-specific months would over-scale years where a rare type appears

pipeline/transform/crime_hotspot_tiles.py

@@ -17,7 +17,7 @@ from pathlib import Path
 import polars as pl
 
 from pipeline.local_temp import local_tmp_dir
-from pipeline.transform.crime import find_street_crime_csvs
+from pipeline.transform.crime import LEGACY_CRIME_TYPE_ALIASES, find_street_crime_csvs
 
 
 def _latest_months(crime_dir: Path, month_count: int) -> list[str]:
@@ -46,8 +46,21 @@ def _require_tippecanoe() -> str:
     return executable
 
 
-def _write_geojsonseq(csvs: list[Path], output_path: Path) -> int:
-    df = (
+def _write_geojsonseq(csvs: list[Path], output_path: Path) -> tuple[int, int]:
+    """Write one weighted GeoJSON point per distinct (anchor, month, type).
+
+    Returns ``(feature_count, incident_count)``. police.uk snaps every incident
+    to a shared "map point" anchor, so many incidents land on the exact same
+    coordinate. Collapsing them into one feature carrying ``count`` (the number
+    of incidents) keeps the per-crime-type and per-month filters intact while
+    turning each hotspot into a single high-weight point. That matters because
+    tippecanoe's ``--drop-densest-as-needed`` thins *feature density*, not
+    weight: with one feature per row the busiest streets were silently deleted;
+    with one weighted feature per anchor those hotspots survive and the dropped
+    detail is only redundant duplicate points. The heatmap reads ``count`` as
+    its weight.
+    """
+    grouped = (
         pl.scan_csv(
             csvs,
             schema_overrides={
@@ -67,11 +80,19 @@ def _write_geojsonseq(csvs: list[Path], output_path: Path) -> int:
         .drop_nulls(["lon", "lat"])
         .filter(pl.col("lon").is_between(-9.5, 5.0))
         .filter(pl.col("lat").is_between(49.0, 57.0))
+        # Canonicalise any legacy pre-2014 type names so the heatmap's crime_type
+        # values always match the frontend's canonical filter list (a no-op for
+        # the recent months this overlay normally covers).
+        .with_columns(pl.col("crime_type").replace(LEGACY_CRIME_TYPE_ALIASES))
+        .group_by("lon", "lat", "month", "crime_type")
+        .len()
+        .rename({"len": "count"})
         .collect(engine="streaming")
     )
 
+    incident_count = int(grouped["count"].sum())
     with output_path.open("w") as file:
-        for row in df.iter_rows(named=True):
+        for row in grouped.iter_rows(named=True):
             feature = {
                 "type": "Feature",
                 "geometry": {
@@ -79,15 +100,15 @@ def _write_geojsonseq(csvs: list[Path], output_path: Path) -> int:
                     "coordinates": [row["lon"], row["lat"]],
                 },
                 "properties": {
-                    "count": 1,
-                    "weight": 1,
+                    "count": row["count"],
+                    "weight": row["count"],
                     "month": row["month"],
                     "crime_type": row["crime_type"],
                 },
             }
             file.write(json.dumps(feature, separators=(",", ":")) + "\n")
 
-    return df.height
+    return grouped.height, incident_count
 
 
 def build_crime_hotspot_tiles(
@@ -104,9 +125,10 @@ def build_crime_hotspot_tiles(
 
     with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as tmp:
         ndjson_path = Path(tmp) / "crime_hotspots.geojsonseq"
-        feature_count = _write_geojsonseq(csvs, ndjson_path)
+        feature_count, incident_count = _write_geojsonseq(csvs, ndjson_path)
         print(
-            f"Writing {feature_count:,} approximate crime heatmap points "
+            f"Writing {feature_count:,} weighted crime heatmap points "
+            f"({incident_count:,} incidents) "
             f"from {min(selected_months)} to {max(selected_months)}"
         )
 

pipeline/transform/crime_spatial.py

@@ -0,0 +1,475 @@
+"""Aggregate police.uk street crime to postcodes by spatial proximity.
+
+Instead of attributing each incident to its published LSOA code, this transform
+counts the anonymised incident *points* that fall within ``buffer_m`` (default
+100m) of each postcode's boundary polygon (the polygon buffered outward). A point
+inside several overlapping buffers counts for each postcode -- the same
+multiplicity the tree-density filter uses for features near more than one
+postcode. The wide 100m buffer deliberately smooths police.uk's snap-to-grid
+coordinates, which would otherwise make the count hypersensitive to which side of
+a narrow line a shared "map point" anchor happened to land on.
+
+Counts are **area-normalised**: each postcode's count is divided by its buffered
+catchment area and rescaled by the median catchment area, so the metric reflects
+crime *density* rather than how much ground the buffer sweeps (a median-sized
+catchment is left unchanged; a large rural postcode is no longer inflated simply
+for covering more of the map). Normalising by the buffered area -- the region
+that actually collects points -- rather than the raw polygon keeps tiny unit
+postcodes from being over-inflated by the fixed buffer-ring floor. The headline
+``"{type} (avg/yr)"`` is the simple mean of the per-year annualised counts, so it
+equals the average of the by-year chart bars.
+
+Outputs mirror the old LSOA transform's shape but are keyed on ``postcode``:
+
+* ``crime_by_postcode.parquet`` -- ``postcode`` + ``"{type} (avg/yr)"`` columns.
+* ``crime_by_postcode_by_year.parquet`` -- ``postcode`` + ``"{type} (by year)"``
+  nested ``list[struct{year, count}]`` columns, with Serious/Minor rollups.
+
+Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
+points", not true locations, and a share of rows have no coordinate at all
+(dropped here). Spatial totals are therefore fuzzier than the old LSOA-tagged
+counts -- by design, not a regression.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import numpy as np
+import polars as pl
+import shapely
+from pyproj import Transformer
+
+from pipeline.transform.crime import (
+    LEGACY_CRIME_TYPE_ALIASES,
+    MINOR_CRIME_TYPES,
+    SERIOUS_CRIME_TYPES,
+    find_street_crime_csvs,
+)
+from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
+
+# Serious types first so column order is stable and self-documenting.
+ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
+
+DEFAULT_BUFFER_M = 100.0
+MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
+
+# Generous GB bounds; points outside fall in no English postcode anyway, but
+# filtering first keeps the WGS84->BNG transform out of its undefined region.
+LON_BOUNDS = (-9.5, 2.5)
+LAT_BOUNDS = (49.0, 61.5)
+
+# Read CSVs in chunks of files to bound peak memory while keeping the STRtree
+# query vectorised over a useful number of points.
+_CSV_BATCH = 64
+
+
+def _month_calendar(csvs: list[Path]) -> tuple[list[int], dict[int, int], int]:
+    """Derive annualisation denominators from the monthly directory names.
+
+    Each police.uk file lives under ``{crime_dir}/{YYYY-MM}/...`` and holds that
+    month's incidents, so the set of month directories is the set of observed
+    months. Returns the sorted distinct years, months-observed-per-year, and the
+    total month count (the avg/yr denominator).
+    """
+    months = sorted(
+        {path.parent.name for path in csvs if MONTH_DIR_RE.fullmatch(path.parent.name)}
+    )
+    if not months:
+        raise ValueError("No valid YYYY-MM month directories found among crime CSVs")
+
+    months_in_year: dict[int, int] = {}
+    for month in months:
+        year = int(month[:4])
+        months_in_year[year] = months_in_year.get(year, 0) + 1
+
+    years = sorted(months_in_year)
+    return years, months_in_year, len(months)
+
+
+def _build_tree(
+    polygons: np.ndarray, buffer_m: float
+) -> tuple[np.ndarray, shapely.STRtree]:
+    """Buffer postcode polygons outward by ``buffer_m`` and index them.
+
+    Buffer index == postcode index. Geometries that fail to buffer are replaced
+    with an empty polygon so the index stays aligned; they simply never match.
+    """
+    buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
+    broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
+    if broken.any():
+        print(f"  {int(broken.sum()):,} postcode buffers unusable; left empty")
+        buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
+    return buffers, shapely.STRtree(buffers)
+
+
+def _accumulate_counts(
+    csvs: list[Path],
+    tree: shapely.STRtree,
+    type_to_idx: dict[str, int],
+    year_to_idx: dict[int, int],
+    transformer: Transformer,
+    counts: np.ndarray,
+) -> None:
+    """Stream the crime CSVs, counting points-in-buffer per (postcode, type, year)."""
+    schema = {
+        "Longitude": pl.Float64,
+        "Latitude": pl.Float64,
+        "Month": pl.Utf8,
+        "Crime type": pl.Utf8,
+    }
+    years = list(year_to_idx)
+    total_points = 0
+    total_matches = 0
+    total_dropped = 0
+    unknown_type_counts: dict[str, int] = {}
+
+    for start in range(0, len(csvs), _CSV_BATCH):
+        batch = csvs[start : start + _CSV_BATCH]
+        frame = (
+            pl.scan_csv(
+                batch,
+                schema_overrides=schema,
+                ignore_errors=True,
+            )
+            .select("Longitude", "Latitude", "Month", "Crime type")
+            # strict=False: a single malformed Month drops only that row instead
+            # of aborting the whole build (a non-numeric year becomes null and is
+            # filtered out by the year membership check below).
+            .with_columns(
+                pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year")
+            )
+            .filter(
+                pl.col("Longitude").is_not_null()
+                & pl.col("Latitude").is_not_null()
+                & pl.col("Longitude").is_between(*LON_BOUNDS)
+                & pl.col("Latitude").is_between(*LAT_BOUNDS)
+                & pl.col("Crime type").is_not_null()
+                & (pl.col("Crime type") != "")
+                & pl.col("year").is_in(years)
+            )
+            # Canonicalise legacy pre-2014 crime-type names ("Violent crime",
+            # "Public disorder and weapons") to their current equivalents before
+            # indexing, so ~1.9M historical incidents are counted instead of
+            # dropped. `.replace` leaves current types unchanged.
+            .with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
+            # Map crime types to indices with default=None so an unrecognised
+            # type yields a null index we can *report* rather than silently drop
+            # (the legacy LSOA path surfaced unknown types via its dynamic pivot).
+            .with_columns(
+                pl.col("Crime type")
+                .replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
+                .alias("tidx"),
+                pl.col("year")
+                .replace_strict(year_to_idx, return_dtype=pl.Int32)
+                .alias("yidx"),
+            )
+            .select("Longitude", "Latitude", "Crime type", "tidx", "yidx")
+            .collect(engine="streaming")
+        )
+
+        if frame.height == 0:
+            continue
+
+        unknown = frame.filter(pl.col("tidx").is_null())
+        if unknown.height:
+            for name, cnt in unknown.group_by("Crime type").len().iter_rows():
+                unknown_type_counts[name] = unknown_type_counts.get(name, 0) + cnt
+            frame = frame.filter(pl.col("tidx").is_not_null())
+            if frame.height == 0:
+                continue
+
+        lon = frame["Longitude"].to_numpy()
+        lat = frame["Latitude"].to_numpy()
+        tidx = frame["tidx"].to_numpy()
+        yidx = frame["yidx"].to_numpy()
+
+        x, y = transformer.transform(lon, lat)
+        finite = np.isfinite(x) & np.isfinite(y)
+        total_dropped += int((~finite).sum())
+        if not finite.any():
+            continue
+        x, y, tidx, yidx = x[finite], y[finite], tidx[finite], yidx[finite]
+        total_points += x.size
+
+        points = shapely.points(x, y)
+        point_index, postcode_index = tree.query(points, predicate="intersects")
+        if point_index.size:
+            np.add.at(
+                counts,
+                (postcode_index, tidx[point_index], yidx[point_index]),
+                1,
+            )
+            total_matches += point_index.size
+
+        print(
+            f"  files {start + len(batch):,}/{len(csvs):,}: "
+            f"{total_points:,} located points, {total_matches:,} postcode matches"
+        )
+
+    if total_dropped:
+        print(f"Dropped {total_dropped:,} points outside the BNG transform domain")
+    if unknown_type_counts:
+        total_unknown = sum(unknown_type_counts.values())
+        listed = ", ".join(
+            f"{name!r} ({cnt:,})"
+            for name, cnt in sorted(
+                unknown_type_counts.items(), key=lambda kv: kv[1], reverse=True
+            )
+        )
+        print(
+            f"WARNING: dropped {total_unknown:,} incidents with crime types not in "
+            f"ALL_CRIME_TYPES (taxonomy is stale -- update SERIOUS/MINOR_CRIME_TYPES): "
+            f"{listed}",
+            file=sys.stderr,
+        )
+
+
+def _rollup_long(
+    long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
+) -> pl.DataFrame:
+    """Sum per-year annualised counts across ``types`` into a single rollup."""
+    return (
+        long.filter(pl.col("Crime type").is_in(list(types)))
+        .group_by("postcode", "year")
+        .agg(pl.col("count").sum().round(1).alias("count"))
+        .with_columns(pl.lit(rollup_name).alias("Crime type"))
+        .select("postcode", "Crime type", "year", "count")
+    )
+
+
+def _write_avg_yr(
+    postcodes: np.ndarray,
+    counts: np.ndarray,
+    years: list[int],
+    months_in_year: dict[int, int],
+    norm: np.ndarray,
+    output_path: Path,
+) -> None:
+    """Write ``postcode`` + ``"{type} (avg/yr)"`` density-normalised averages.
+
+    The headline figure is the **simple mean of the per-year annualised counts**
+    (each year scaled to a 12-month equivalent), so it equals the average of the
+    by-year chart bars instead of a month-weighted pooled rate. Each postcode's
+    value is then multiplied by ``norm`` (median_area / buffered catchment area)
+    so the metric is a density rather than a footprint-inflated raw count.
+    """
+    months = np.array([months_in_year[year] for year in years], dtype=np.float64)
+    per_year = counts.astype(np.float64) * 12.0 / months[None, None, :]
+    # Average over the years *this postcode* actually has incidents of *this
+    # type* -- the same per-(postcode, type) x-span the by-year chart plots
+    # (server-rs/.../crime_by_year.rs), so the headline equals the mean of the
+    # by-year bars. Dividing by a global years-present count (years a type
+    # appeared anywhere in England) would deflate postcodes whose incidents
+    # cluster in only a few years of the ~13-year window.
+    years_present = np.clip((counts > 0).sum(axis=2), 1, None).astype(np.float64)
+    avg = per_year.sum(axis=2) / years_present  # (n_postcodes, n_types)
+    avg = np.round(avg * norm[:, None], 1).astype(np.float32)
+
+    data: dict[str, np.ndarray] = {"postcode": postcodes}
+    for type_idx, name in enumerate(ALL_CRIME_TYPES):
+        data[f"{name} (avg/yr)"] = avg[:, type_idx]
+
+    # Serious/Minor rollup headlines, computed the SAME way as the by-year rollup
+    # bars (_write_by_year/_rollup_long): sum the rollup's types per year, then
+    # average over the years in which ANY of those types occurred. This keeps the
+    # headline equal to the mean of the "Serious/Minor crime (by year)" bars.
+    # Summing the per-type avg/yr values instead (as the merge previously did)
+    # divides each type by its OWN years-present and overstates the rollup when a
+    # postcode's serious/minor types occur in disjoint years.
+    for rollup_name, rollup_types in (
+        ("Serious crime", SERIOUS_CRIME_TYPES),
+        ("Minor crime", MINOR_CRIME_TYPES),
+    ):
+        rollup_idx = [ALL_CRIME_TYPES.index(name) for name in rollup_types]
+        rollup_counts = counts[:, rollup_idx, :].sum(axis=1)  # (n_postcodes, n_years)
+        rollup_per_year = per_year[:, rollup_idx, :].sum(axis=1)
+        rollup_years_present = np.clip(
+            (rollup_counts > 0).sum(axis=1), 1, None
+        ).astype(np.float64)
+        rollup_avg = rollup_per_year.sum(axis=1) / rollup_years_present
+        data[f"{rollup_name} (avg/yr)"] = np.round(rollup_avg * norm, 1).astype(
+            np.float32
+        )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    pl.DataFrame(data).write_parquet(output_path, compression="zstd")
+    print(f"Wrote postcode crime averages: {output_path}")
+
+
+def _write_by_year(
+    postcodes: np.ndarray,
+    counts: np.ndarray,
+    years: list[int],
+    months_in_year: dict[int, int],
+    norm: np.ndarray,
+    output_path: Path,
+) -> None:
+    """Write nested ``"{type} (by year)"`` series plus Serious/Minor rollups.
+
+    Per-year counts are area-normalised by the same ``norm`` (median_area /
+    buffered catchment area) factor applied to the avg/yr headline, so the chart
+    bars and the headline figure remain mutually consistent.
+    """
+    months = np.array([months_in_year[year] for year in years], dtype=np.float64)
+    annual = np.round(
+        counts.astype(np.float64) * 12.0 / months[None, None, :] * norm[:, None, None],
+        1,
+    )
+
+    pc_i, ty_i, yr_i = np.nonzero(counts)
+    if pc_i.size == 0:
+        raise ValueError("No crime points matched any postcode buffer")
+
+    type_names = np.array(ALL_CRIME_TYPES, dtype=object)
+    year_values = np.array(years, dtype=np.int32)
+    long = pl.DataFrame(
+        {
+            "postcode": postcodes[pc_i],
+            "Crime type": type_names[ty_i],
+            "year": year_values[yr_i],
+            "count": annual[pc_i, ty_i, yr_i].astype(np.float32),
+        }
+    )
+
+    serious = _rollup_long(long, SERIOUS_CRIME_TYPES, "Serious crime")
+    minor = _rollup_long(long, MINOR_CRIME_TYPES, "Minor crime")
+    combined = pl.concat([long, serious, minor])
+
+    by_type = (
+        combined.sort("year")
+        .group_by("postcode", "Crime type")
+        .agg(pl.struct("year", "count").alias("series"))
+    )
+    wide = by_type.pivot(on="Crime type", index="postcode", values="series")
+    type_cols = [c for c in wide.columns if c != "postcode"]
+    wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    wide.write_parquet(output_path, compression="zstd")
+    print(f"Wrote postcode crime by-year series: {output_path}  {wide.shape}")
+
+
+def transform_crime_spatial(
+    crime_dir: Path,
+    boundaries_dir: Path,
+    output_path: Path,
+    by_year_output_path: Path,
+    buffer_m: float = DEFAULT_BUFFER_M,
+    max_postcodes: int | None = None,
+    max_files: int | None = None,
+) -> None:
+    csvs, ignored_csv_count = find_street_crime_csvs(crime_dir)
+    if not csvs:
+        raise FileNotFoundError(f"No street crime CSV files found in {crime_dir}")
+    if max_files is not None:
+        csvs = csvs[:max_files]
+
+    years, months_in_year, valid_month_count = _month_calendar(csvs)
+    print(
+        f"Found {len(csvs):,} street crime CSVs across {valid_month_count} months "
+        f"({years[0]}-{years[-1]})"
+        + (f" (ignored {ignored_csv_count} non-street CSVs)" if ignored_csv_count else "")
+    )
+
+    postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
+
+    print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
+    buffers, tree = _build_tree(polygons, buffer_m)
+
+    # Area-normalisation factor (median_area / catchment_area): divides out the
+    # size of each postcode's catchment so the count measures crime density, not
+    # how much ground the buffer sweeps. We normalise by the *buffered* area --
+    # the region that actually collects points -- rather than the raw polygon, so
+    # a tiny unit postcode isn't over-inflated by the fixed buffer-ring floor.
+    # Buffers are in EPSG:27700, so shapely.area is in m^2.
+    areas = shapely.area(buffers).astype(np.float64)
+    usable_area = np.isfinite(areas) & (areas > 0)
+    if not usable_area.any():
+        raise ValueError("No postcode buffers have a positive area to normalise by")
+    median_area = float(np.median(areas[usable_area]))
+    norm = np.zeros(len(postcodes), dtype=np.float64)
+    norm[usable_area] = median_area / areas[usable_area]
+    print(
+        f"Area-normalising to median catchment area {median_area:,.0f} m^2 "
+        f"({int(usable_area.sum()):,}/{len(areas):,} postcodes have usable area)"
+    )
+
+    type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
+    year_to_idx = {year: idx for idx, year in enumerate(years)}
+    counts = np.zeros((len(postcodes), len(ALL_CRIME_TYPES), len(years)), dtype=np.int32)
+
+    transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
+    _accumulate_counts(csvs, tree, type_to_idx, year_to_idx, transformer, counts)
+
+    _write_avg_yr(postcodes, counts, years, months_in_year, norm, output_path)
+    _write_by_year(postcodes, counts, years, months_in_year, norm, by_year_output_path)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Count police.uk crime points within 50m of each postcode boundary"
+    )
+    parser.add_argument(
+        "--input",
+        type=Path,
+        default=Path("property-data/crime"),
+        help="Directory containing police.uk street crime CSVs",
+    )
+    parser.add_argument(
+        "--boundaries",
+        type=Path,
+        default=Path("property-data/postcode_boundaries/units"),
+        help="Directory of per-district postcode boundary GeoJSONs",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        required=True,
+        help="Output parquet: postcode + '{type} (avg/yr)' columns",
+    )
+    parser.add_argument(
+        "--output-by-year",
+        type=Path,
+        required=True,
+        help="Output parquet: postcode + nested '{type} (by year)' columns",
+    )
+    parser.add_argument(
+        "--buffer-m",
+        type=float,
+        default=DEFAULT_BUFFER_M,
+        help="Outward buffer (metres) added to each postcode boundary",
+    )
+    parser.add_argument(
+        "--max-postcodes",
+        type=int,
+        default=None,
+        help="Testing only: process the first N postcodes",
+    )
+    parser.add_argument(
+        "--max-files",
+        type=int,
+        default=None,
+        help="Testing only: process the first N monthly CSV files",
+    )
+    args = parser.parse_args()
+
+    if args.buffer_m <= 0:
+        raise SystemExit("--buffer-m must be greater than zero")
+
+    transform_crime_spatial(
+        crime_dir=args.input,
+        boundaries_dir=args.boundaries,
+        output_path=args.output,
+        by_year_output_path=args.output_by_year,
+        buffer_m=args.buffer_m,
+        max_postcodes=args.max_postcodes,
+        max_files=args.max_files,
+    )
+
+
+if __name__ == "__main__":
+    main()

pipeline/transform/join_epc_pp.py

@@ -18,14 +18,53 @@ from ..utils import (
     normalize_postcode_key,
 )
 
-
 pl.Config.set_tbl_cols(-1)
 
 RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7}
 MIN_PRICE = 50_000
+
+# Plausible construction-year range; band-derived years outside it (e.g. OCR
+# noise like 1012 or 2202) are nulled rather than published.
+MIN_BUILD_YEAR = 1700
+MAX_BUILD_YEAR = 2030
+
+
+def epc_band_to_year(band: pl.Expr) -> pl.Expr:
+    """Map an EPC construction age band to a single representative build year.
+
+    EPC age bands are ranges (e.g. ``1950-1966``); we use the band MIDPOINT
+    (1958) rather than the lower bound, which previously biased every band-derived
+    year ~10-15 years too young. Open-ended lower bands (``before 1900``) are too
+    wide to pin to a year and return null. Single-year / ``... onwards`` bands use
+    that year. Already-numeric inputs (a year produced by an earlier call) pass
+    through unchanged. Years outside [MIN_BUILD_YEAR, MAX_BUILD_YEAR] are nulled.
+    """
+    text = (
+        band.cast(pl.Utf8)
+        .str.replace("England and Wales: ", "")
+        .str.replace(" onwards", "")
+    )
+    low = text.str.extract(r"(\d{4})", 1).cast(pl.Int32, strict=False)
+    high = text.str.extract(r"(\d{4})\D+(\d{4})", 2).cast(pl.Int32, strict=False)
+    year = (
+        pl.when(text.str.starts_with("before "))
+        .then(None)
+        .when(high.is_not_null())
+        .then(((low + high) / 2).round(0).cast(pl.Int32))
+        .otherwise(low)
+    )
+    return (
+        pl.when((year >= MIN_BUILD_YEAR) & (year <= MAX_BUILD_YEAR))
+        .then(year)
+        .otherwise(None)
+        .cast(pl.UInt16, strict=False)
+    )
+
+
 EPC_SOURCE_COLUMNS = [
     "address",
     "postcode",
+    "uprn",
     "current_energy_rating",
     "potential_energy_rating",
     "property_type",
@@ -57,6 +96,8 @@ def _select_epc_columns(raw: pl.LazyFrame) -> pl.LazyFrame:
         raw.select(
             _clean_string("address").alias("epc_address"),
             _clean_string("postcode").str.to_uppercase().alias("epc_postcode"),
+            # UPRN keys an exact listing->EPC join downstream (~99% populated).
+            _clean_string("uprn").alias("uprn"),
             _clean_string("current_energy_rating")
             .str.to_uppercase()
             .alias("current_energy_rating"),
@@ -65,7 +106,14 @@ def _select_epc_columns(raw: pl.LazyFrame) -> pl.LazyFrame:
             .alias("potential_energy_rating"),
             _clean_string("property_type").alias("epc_property_type"),
             _clean_string("built_form").alias("built_form"),
-            _clean_string("inspection_date").alias("inspection_date"),
+            # Parse to a real Date once (unparseable/blank -> null) so dedup can
+            # sort newest-first with nulls_last and _event_year can use dt.year();
+            # a lexicographic string sort would let a null/garbled date win under
+            # Polars' default nulls-first descending order. EPC inspection dates
+            # are ISO (YYYY-MM-DD).
+            _clean_string("inspection_date")
+            .str.to_date(format="%Y-%m-%d", strict=False)
+            .alias("inspection_date"),
             _clean_number("total_floor_area", pl.Float64).alias("total_floor_area"),
             _clean_number("number_habitable_rooms", pl.Int16).alias(
                 "number_habitable_rooms"
@@ -206,9 +254,11 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
         normalize_postcode_key(pl.col("epc_postcode")).alias("_epc_match_postcode"),
     )
 
-    # Dedup fork: keep latest certificate per property (existing logic)
+    # Dedup fork: keep latest certificate per property. inspection_date is a typed
+    # Date (see _select_epc_columns); nulls_last keeps a real-dated cert ahead of a
+    # null/unparseable-dated one so the genuinely newest certificate is chosen.
     epc = (
-        epc_base.sort("inspection_date", descending=True)
+        epc_base.sort("inspection_date", descending=True, nulls_last=True)
         .group_by("_epc_match_address", "_epc_match_postcode")
         .first()
         .drop("tenure")
@@ -262,11 +312,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
         )
         .filter(pl.col("_event").is_not_null())
         .with_columns(
-            pl.col("inspection_date")
-            .cast(pl.String)
-            .str.slice(0, 4)
-            .cast(pl.Int32)
-            .alias("_event_year"),
+            pl.col("inspection_date").dt.year().cast(pl.Int32).alias("_event_year"),
         )
         .group_by("_epc_match_address", "_epc_match_postcode")
         .agg(
@@ -324,6 +370,16 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     }
     duration_map = {"F": "Freehold", "L": "Leasehold"}
 
+    # price >= MIN_PRICE and ppd_category == "A" (standard open-market sale) are
+    # VALUE-QUALITY filters: they gate the price aggregations only. Category B
+    # entries (repossessions, bulk/portfolio, power-of-sale transfers) and sub-MIN
+    # sales must not pollute latest_price / historical_prices (and the downstream
+    # price-per-sqm feature), but they MUST still count for first_transfer_date /
+    # old_new so a new-build's genuine earliest transfer year is preserved.
+    price_ok = pl.col("price") >= MIN_PRICE
+    category_ok = pl.col("ppd_category") == "A"
+    quality_ok = price_ok & category_ok
+
     price_paid = (
         pl.scan_parquet(price_paid_path)
         .select(
@@ -340,9 +396,9 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
             "town_city",
             pl.col("duration").replace(duration_map),
             "old_new",
+            "ppd_category",
         )
         .filter(pl.col("pp_property_type") != "Other")
-        .filter(pl.col("price") >= MIN_PRICE)
         .with_columns(
             pl.concat_str(
                 [pl.col("saon"), pl.col("paon"), pl.col("street")],
@@ -367,18 +423,26 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
             pl.col("postcode").last(),
             pl.col("_pp_match_address").last(),
             pl.col("_pp_match_postcode").last(),
+            # Price aggregations are restricted to quality-passing sales.
             pl.struct(
                 pl.col("date_of_transfer").dt.year().alias("year"),
                 pl.col("date_of_transfer").dt.month().cast(pl.UInt8).alias("month"),
                 "price",
-            ).alias("historical_prices"),
+            )
+            .filter(quality_ok)
+            .alias("historical_prices"),
             pl.col("pp_property_type").last(),
             pl.col("duration").last(),
-            pl.col("price").last().alias("latest_price"),
-            pl.col("date_of_transfer").last(),
+            pl.col("price").filter(quality_ok).last().alias("latest_price"),
+            pl.col("date_of_transfer").filter(quality_ok).last(),
+            # first_transfer_date / old_new reflect the genuine earliest transfer
+            # over the full per-group transaction stream (not value-filtered).
             pl.col("date_of_transfer").first().alias("first_transfer_date"),
             pl.col("old_new").first(),
         )
+        # Preserve the property universe: previously a property needed >=1 sale
+        # >=MIN_PRICE to form a group, so drop groups with no quality-passing sale.
+        .filter(pl.col("latest_price").is_not_null())
     )
 
     print("Price paid dataset")
@@ -407,13 +471,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
 
     # For new-builds (old_new == "Y"), use the first transaction date year as
     # the exact construction date; otherwise fall back to the EPC age band.
-    epc_band_year = (
-        pl.col("construction_age_band")
-        .str.replace("England and Wales: ", "")
-        .str.replace(" onwards", "")
-        .str.extract(r"(\d{4})", 1)
-        .cast(pl.UInt16, strict=False)
-    )
+    epc_band_year = epc_band_to_year(pl.col("construction_age_band"))
     transfer_year = (
         pl.col("first_transfer_date").dt.year().cast(pl.UInt16, strict=False)
     )

pipeline/transform/noise_overlay_tiles.py

@@ -164,19 +164,39 @@ def _read_noise_tile(
 
     for info in candidates:
         with rasterio.open(info.path) as source:
+            # The Defra rasters encode genuine "quiet / below threshold" as the
+            # value 0.0 (only -96.0 is true nodata). Mask both BEFORE
+            # reprojecting so resampling never blends a 0 cell into an adjacent
+            # loud corridor and fabricates a halo of intermediate dB.
+            #
+            # Lden values are dB (a logarithmic scale), so bilinear resampling
+            # would arithmetically average neighbouring dB cells, which is
+            # acoustically wrong (it diluted a 75 dB peak to ~53 dB in tests)
+            # and inconsistent with the postcode sampler. Use Resampling.max:
+            # it preserves peak corridors, never invents an intermediate dB
+            # between a masked (NaN) quiet cell and a loud one, and mirrors the
+            # max semantics of sample_noise_at_postcodes.
+            src_arr = source.read(1).astype(np.float32)
+            nodata = source.nodata
+            invalid = ~np.isfinite(src_arr) | (src_arr <= 0)
+            if nodata is not None:
+                invalid |= np.isclose(
+                    src_arr, np.float32(nodata), rtol=1e-5, atol=1e-5
+                )
+            src_arr = np.where(invalid, np.float32("nan"), src_arr)
             tile = np.full((tile_size, tile_size), np.nan, dtype=np.float32)
             reproject(
-                source=rasterio.band(source, 1),
+                source=src_arr,
                 destination=tile,
                 src_transform=source.transform,
                 src_crs=source.crs,
-                src_nodata=source.nodata if source.nodata is not None else 0,
+                src_nodata=float("nan"),
                 dst_transform=from_bounds(
                     left, bottom, right, top, tile_size, tile_size
                 ),
                 dst_crs=WEB_MERCATOR_CRS,
                 dst_nodata=np.nan,
-                resampling=Resampling.bilinear,
+                resampling=Resampling.max,
             )
 
         tile[~np.isfinite(tile) | (tile <= 0)] = np.nan
@@ -376,7 +396,7 @@ def main() -> None:
         "--pmtiles-bin", type=Path, default=Path("property-data/pmtiles")
     )
     parser.add_argument("--pmtiles-version", default="1.22.3")
-    parser.add_argument("--min-zoom", type=int, default=13)
+    parser.add_argument("--min-zoom", type=int, default=12)
     parser.add_argument("--max-zoom", type=int, default=14)
     parser.add_argument("--tile-size", type=int, default=256)
     args = parser.parse_args()

pipeline/transform/poi_proximity.py

@@ -12,11 +12,19 @@ from pipeline.utils.poi_counts import count_pois_per_postcode, min_distance_per_
 
 # POI category groups for proximity counting (2km radius).
 # Names must match the friendly names produced by transform_poi.py / naptan.py.
+# "groceries" is filled in dynamically by _groceries_categories() because the
+# GEOLYTIX dataset stores the brand (e.g. "Tesco", "Aldi") in `category` rather
+# than the literal "Supermarket"; counting only the OSM strings here severely
+# understates the metric. See _groceries_categories below.
 POI_GROUPS_2KM = {
     "restaurants": ["Restaurant", "Fast Food"],
-    "groceries": ["Greengrocer", "Supermarket", "Convenience Store"],
 }
 
+# POI group whose members are counted for the static "groceries" 2km metric.
+# Covers both the OSM grocery categories (Supermarket, Convenience Store,
+# Greengrocer, ...) and the GEOLYTIX brand categories (Tesco, Aldi, ...).
+GROCERIES_GROUP = "Groceries"
+
 # OS Open Greenspace function types used for park counts and distance calculation.
 # Uses the authoritative OS dataset instead of OSM point POIs for better coverage
 # of green spaces that are only mapped as polygons in OSM.
@@ -41,6 +49,26 @@ def _poi_category_slug(category: str) -> str:
     return slug or "poi"
 
 
+def _groceries_categories(pois: pl.DataFrame) -> list[str]:
+    """Return the distinct `category` values for the Groceries group.
+
+    `count_pois_per_postcode` matches POIs on `category`, but the authoritative
+    GEOLYTIX grocery dataset stores the brand name there (e.g. "Tesco", "Aldi")
+    with group "Groceries"; it never emits the literal "Supermarket". Collecting
+    every Groceries category captures both the OSM strings and the brand names.
+    """
+    if "group" not in pois.columns:
+        raise ValueError("POI dataframe must include a 'group' column")
+    return (
+        pois.filter(pl.col("group") == GROCERIES_GROUP)
+        .select("category")
+        .unique()
+        .sort("category")
+        .to_series()
+        .to_list()
+    )
+
+
 def _build_poi_category_groups(
     pois: pl.DataFrame,
 ) -> tuple[dict[str, list[str]], dict[str, str]]:
@@ -122,9 +150,15 @@ def main():
     pois = pl.read_parquet(args.pois)
     poi_category_groups, poi_display_names = _build_poi_category_groups(pois)
 
-    # Count static amenity groups within 2km.
+    # Count static amenity groups within 2km. "groceries" is matched against
+    # every Groceries category (OSM strings + GEOLYTIX brand names) so that
+    # postcodes ringed by GEOLYTIX-only chains (Tesco, Aldi, ...) are counted.
+    groups_2km = {
+        **POI_GROUPS_2KM,
+        "groceries": _groceries_categories(pois),
+    }
     counts_2km = count_pois_per_postcode(
-        postcodes, pois, groups=POI_GROUPS_2KM, radius_km=2
+        postcodes, pois, groups=groups_2km, radius_km=2
     )
 
     # Dynamic amenity filters: nearest distance plus counts within 2km and 5km for

pipeline/transform/postcode_boundaries/README.md

@@ -37,11 +37,11 @@ Pre-allocates numpy arrays at 25M capacity and grows by 1.5x if needed (using in
 
 **Loading** (`inspire.py:load_inspire`): Bboxes and offsets are loaded into RAM (~1.1GB). Coords are memory-mapped — the OS pages them in on demand from the ~3GB file, never loading the whole thing.
 
-**Candidate retrieval** (`inspire.py:get_inspire_candidates`): Given an OA's bounding box, performs a vectorized numpy overlap test against all 24M INSPIRE bboxes — four comparisons broadcast across the entire array. Typically matches 10-500 parcels per OA. Only those matches are materialized as Shapely Polygon objects by reading their coordinate slice from the memory-mapped file. Invalid polygons are repaired with `make_valid`.
+**Candidate retrieval** (`inspire.py:InspireIndex`): A uniform 1km grid index is built once over the 24M parcel bboxes (`build_inspire_index`). Each OA lookup (`InspireIndex.candidates`) gathers parcels from the cells its bounding box covers plus a small overflow list of parcels larger than one cell, then applies the exact bbox overlap test — O(cells + candidates) instead of an O(24M) scan per OA (the old linear scan was ~4h of the run on its own). The candidate set and order are identical to the scan. Typically matches 10-500 parcels per OA, materialized as Shapely Polygon objects by reading their coordinate slice from the memory-mapped file; invalid polygons are repaired with `make_valid`.
 
 ### Phase 3: Processing OAs
 
-The main loop in `__main__.py` iterates through every OA that has both a boundary polygon and UPRNs. For each OA, it retrieves the OA's UPRN points and postcodes.
+The main loop in `__main__.py` (`_process_oas`) iterates through every OA that has both a boundary polygon and UPRNs. For each OA, it retrieves the OA's UPRN points and postcodes. OAs are independent, so the loop fans out across CPU cores with a `fork` process pool (`--workers`, default all CPUs): workers share the big read-only inputs (INSPIRE arrays + coords mmap, UPRN arrays, OA geometries) copy-on-write and return WKB-encoded fragments. Workers slice the UPRN data from plain numpy/Arrow arrays (`extract_uprn_arrays`) rather than polars, avoiding the fork-after-threads hazard of polars' thread pool. Fragment order doesn't affect the output (`merge_fragments` unions per postcode), so the parallel result is identical to single-process.
 
 **Fast path**: If every UPRN in the OA shares the same postcode, the entire OA polygon is assigned to that postcode. No geometry computation needed. This covers the majority of OAs (~70-80%).
 
@@ -77,9 +77,9 @@ The output of `process_oa` is `list[(postcode, polygon)]` — the per-OA fragmen
 
 ### Phase 4: Merging and writing
 
-**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps only the largest polygon — postcodes are contiguous delivery routes, so detached fragments are artifacts.
+**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 3000–5000 m² building blocks) before this change.
 
-**GeoJSON output** (`output.py:write_district_geojson`): Groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`). For each district, converts every postcode polygon from BNG to WGS84 using pyproj, simplifies with 1m tolerance (Douglas-Peucker), rounds coordinates to 6 decimal places (~0.1m precision), and writes a single `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
+**GeoJSON output** (`output.py:write_district_geojson`): Two passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
 
 ## Memory architecture
 
@@ -103,10 +103,10 @@ Key design choices:
 
 ## Key invariants
 
-1. **Every square meter of every OA is assigned to exactly one postcode** — the combination of INSPIRE claiming + Voronoi fills the entire OA, and overlap resolution ensures no double-counting
+1. **No two postcodes cover the same ground in the output** — within an OA the INSPIRE claiming + Voronoi tile it with no overlap, and a final `_resolve_overlaps` partition pass removes the thin overlap strips that the merge buffer + per-postcode simplification introduce across OA seams (measured residual overlap ~0.01% of area)
 2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
 3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
-4. **Output is always single Polygon, never MultiPolygon** — the largest-polygon extraction in both `merge_fragments` and `to_wgs84_geojson` ensures this
+4. **A postcode split across an OA seam keeps all its substantial parts** — `merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
 
 ## Module structure
 

pipeline/transform/postcode_boundaries/__main__.py

@@ -1,12 +1,21 @@
 import argparse
+import multiprocessing as mp
+import os
 from pathlib import Path
 
+import numpy as np
+import shapely
 from shapely.geometry import MultiPolygon, Polygon
 from tqdm import tqdm
 
+from .fragments_cache import (
+    fragments_cache_is_fresh,
+    load_fragments,
+    save_fragments,
+)
 from .inspire import (
+    build_inspire_index,
     cache_inspire,
-    get_inspire_candidates,
     inspire_cache_exists,
     load_inspire,
 )
@@ -14,39 +23,156 @@ from .memory import release_memory
 from .oa_boundaries import load_oa_boundaries
 from .output import merge_fragments, write_district_geojson
 from .process_oa import process_oa
-from .uprn import get_oa_uprns, load_uprns
+from .uprn import extract_uprn_arrays, get_oa_uprns_arrays, load_uprns
+
+Fragment = tuple[str, Polygon | MultiPolygon]
 
 
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Generate postcode boundary polygons from OA + INSPIRE + UPRN data"
-    )
-    parser.add_argument("--uprn", type=Path, required=True, help="UPRN lookup parquet")
-    parser.add_argument(
-        "--oa-boundaries", type=Path, required=True, help="OA boundaries GeoPackage"
-    )
-    parser.add_argument(
-        "--inspire", type=Path, required=True, help="INSPIRE ZIP directory"
-    )
-    parser.add_argument("--output", type=Path, required=True, help="Output directory")
-    parser.add_argument(
-        "--limit", type=int, default=0, help="Process only first N OAs (0=all)"
-    )
-    parser.add_argument(
-        "--greenspace",
-        type=Path,
-        default=None,
-        help="Greenspace/water parquet for boundary trimming (optional)",
-    )
-    args = parser.parse_args()
+def _oa_fragments(
+    oa_code, oa_geoms, east, north, postcodes_arr, offsets, index
+) -> tuple[list[Fragment], bool]:
+    """Process one OA into ``(postcode, geometry)`` fragments.
 
+    Returns ``(fragments, is_single)``; ``is_single`` flags the single-postcode
+    fast path. Shared by the sequential and parallel drivers so both produce
+    identical output. Any failure is re-raised tagged with the OA code so a single
+    bad OA is attributable instead of an anonymous worker abort hours in.
+    """
+    try:
+        oa_geom = oa_geoms[oa_code]
+        points, postcodes = get_oa_uprns_arrays(
+            east, north, postcodes_arr, offsets, oa_code
+        )
+        if len(set(postcodes)) == 1:
+            return [(postcodes[0], oa_geom)], True
+        candidates = index.candidates(oa_geom.bounds)
+        return process_oa(oa_geom, points, postcodes, candidates), False
+    except Exception as exc:
+        raise RuntimeError(f"Failed processing OA {oa_code}: {exc!r}") from exc
+
+
+# Worker-shared state. Populated in the parent before the pool forks; children
+# inherit it copy-on-write (the numpy/Arrow buffers + coords mmap stay shared,
+# never duplicated per worker). Read-only in workers.
+_WORKER_STATE: dict = {}
+
+
+def _process_oa_chunk(oa_codes: list[str]):
+    """Worker: turn a chunk of OA codes into WKB-encoded fragments.
+
+    Geometries are returned as WKB (compact and lossless) rather than pickled
+    Shapely objects, to keep the IPC payload small.
+    """
+    state = _WORKER_STATE
+    frags: list[Fragment] = []
+    single = 0
+    for oa_code in oa_codes:
+        oa_frags, is_single = _oa_fragments(
+            oa_code,
+            state["oa_geoms"],
+            state["east"],
+            state["north"],
+            state["postcodes"],
+            state["offsets"],
+            state["index"],
+        )
+        frags.extend(oa_frags)
+        single += is_single
+
+    if frags:
+        pcs = [pc for pc, _ in frags]
+        wkb = shapely.to_wkb(np.array([g for _, g in frags], dtype=object))
+    else:
+        pcs, wkb = [], np.empty(0, dtype=object)
+    return pcs, wkb, single, len(oa_codes)
+
+
+def _resolve_workers(requested: int) -> int:
+    """Worker count: the explicit value if >0, otherwise all available CPUs."""
+    if requested and requested > 0:
+        return requested
+    try:
+        return max(1, len(os.sched_getaffinity(0)))
+    except AttributeError:
+        return max(1, os.cpu_count() or 1)
+
+
+def _process_oas(
+    oa_codes, oa_geoms, east, north, postcodes_arr, offsets, index, workers
+) -> tuple[list[Fragment], int]:
+    """Drive Phase 3 over every OA, fanning out across `workers` processes.
+
+    OAs are independent, so the loop parallelises cleanly. ``fork`` lets workers
+    share the big read-only inputs (INSPIRE arrays + coords mmap, UPRN arrays, OA
+    geometries) copy-on-write instead of duplicating ~2GB each. Fragment order
+    does not affect the result (``merge_fragments`` unions per postcode), so
+    chunks are collected as they finish. Returns ``(fragments, single_count)``.
+    """
+    all_fragments: list[Fragment] = []
+    single_count = 0
+
+    if workers <= 1 or "fork" not in mp.get_all_start_methods():
+        for oa_code in tqdm(
+            oa_codes, desc="Processing OAs", unit="OA", smoothing=0.01, miniters=100
+        ):
+            oa_frags, is_single = _oa_fragments(
+                oa_code, oa_geoms, east, north, postcodes_arr, offsets, index
+            )
+            all_fragments.extend(oa_frags)
+            single_count += is_single
+        return all_fragments, single_count
+
+    _WORKER_STATE.update(
+        oa_geoms=oa_geoms,
+        east=east,
+        north=north,
+        postcodes=postcodes_arr,
+        offsets=offsets,
+        index=index,
+    )
+    # Many small contiguous chunks → dynamic load balancing across workers (rural
+    # OAs cost far more than urban ones) while preserving mmap read locality.
+    chunk_size = max(1, len(oa_codes) // (workers * 16))
+    chunks = [oa_codes[i : i + chunk_size] for i in range(0, len(oa_codes), chunk_size)]
+    print(f"  Parallel: {workers} workers, {len(chunks)} chunks of ~{chunk_size} OAs")
+
+    ctx = mp.get_context("fork")
+    try:
+        with ctx.Pool(processes=workers) as pool:
+            with tqdm(
+                total=len(oa_codes), desc="Processing OAs", unit="OA", smoothing=0.01
+            ) as bar:
+                for pcs, wkb, single, n_oas in pool.imap_unordered(
+                    _process_oa_chunk, chunks
+                ):
+                    if len(wkb):
+                        all_fragments.extend(zip(pcs, shapely.from_wkb(wkb)))
+                    single_count += single
+                    bar.update(n_oas)
+    finally:
+        # Drop references so Phase 4 doesn't keep the big inputs alive.
+        _WORKER_STATE.clear()
+    return all_fragments, single_count
+
+
+def build_fragments(args: argparse.Namespace) -> list[Fragment]:
+    """Run Phases 1-3: load data, parse INSPIRE, process every OA into fragments.
+
+    Returns the full ``(postcode, geometry)`` fragment list. The large
+    intermediate structures (OA/UPRN/INSPIRE arrays) are locals here, so they are
+    freed as soon as this function returns -- before the fragments are cached or
+    merged.
+    """
     # Phase 1: Load all data
     print("=" * 60)
     print("Phase 1: Loading data")
     print("=" * 60)
 
     oa_geoms = load_oa_boundaries(args.oa_boundaries)
-    uprn_df, uprn_offsets = load_uprns(args.uprn)
+    uprn_df, uprn_offsets = load_uprns(args.uprn, args.arcgis)
+    # Convert UPRNs to fork-shareable numpy/Arrow arrays so parallel workers never
+    # call polars (avoids the fork-after-threads hazard of its rayon pool).
+    uprn_east, uprn_north, uprn_postcodes = extract_uprn_arrays(uprn_df)
 
     # Phase 2: Parse/load INSPIRE
     print()
@@ -58,6 +184,7 @@ def main() -> None:
     if not inspire_cache_exists(inspire_cache_dir):
         cache_inspire(args.inspire, inspire_cache_dir)
     inspire_bboxes, inspire_offsets, inspire_coords = load_inspire(inspire_cache_dir)
+    inspire_index = build_inspire_index(inspire_bboxes, inspire_offsets, inspire_coords)
 
     # Phase 3: Process OAs
     print()
@@ -77,42 +204,86 @@ def main() -> None:
     print(f"  Skipped (no UPRNs): {skipped_no_uprn}")
     print(f"  Skipped (no boundary): {skipped_no_boundary}")
 
-    all_fragments: list[tuple[str, Polygon | MultiPolygon]] = []
-    single_count = 0
-    multi_count = 0
-
-    for oa_code in tqdm(
+    # --limit is a debug mode → force deterministic single-process.
+    workers = 1 if args.limit > 0 else _resolve_workers(args.workers)
+    all_fragments, single_count = _process_oas(
         oa_codes_with_data,
-        desc="Processing OAs",
-        unit="OA",
-        smoothing=0.01,
-        miniters=100,
-    ):
-        oa_geom = oa_geoms[oa_code]
-        points, postcodes = get_oa_uprns(uprn_df, uprn_offsets, oa_code)
-
-        if len(set(postcodes)) == 1:
-            # Fast path: entire OA = one postcode
-            all_fragments.append((postcodes[0], oa_geom))
-            single_count += 1
-            continue
-
-        # Get INSPIRE candidates via bbox pre-filter
-        candidates = get_inspire_candidates(
-            oa_geom.bounds, inspire_bboxes, inspire_offsets, inspire_coords
+        oa_geoms,
+        uprn_east,
+        uprn_north,
+        uprn_postcodes,
+        uprn_offsets,
+        inspire_index,
+        workers,
     )
-
-        fragments = process_oa(oa_geom, points, postcodes, candidates)
-        all_fragments.extend(fragments)
-        multi_count += 1
+    multi_count = len(oa_codes_with_data) - single_count
 
     print(f"\n  Single-postcode OAs (fast path): {single_count}")
     print(f"  Multi-postcode OAs (INSPIRE+Voronoi): {multi_count}")
     print(f"  Total fragments: {len(all_fragments)}")
 
-    # Free data no longer needed
-    del oa_geoms, uprn_df, uprn_offsets
-    del inspire_bboxes, inspire_offsets, inspire_coords
+    return all_fragments
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Generate postcode boundary polygons from OA + INSPIRE + UPRN data"
+    )
+    parser.add_argument("--uprn", type=Path, required=True, help="UPRN lookup parquet")
+    parser.add_argument(
+        "--arcgis",
+        type=Path,
+        default=None,
+        help="Optional ArcGIS postcode parquet used to remap terminated postcodes",
+    )
+    parser.add_argument(
+        "--oa-boundaries", type=Path, required=True, help="OA boundaries GeoPackage"
+    )
+    parser.add_argument(
+        "--inspire", type=Path, required=True, help="INSPIRE ZIP directory"
+    )
+    parser.add_argument("--output", type=Path, required=True, help="Output directory")
+    parser.add_argument(
+        "--limit", type=int, default=0, help="Process only first N OAs (0=all)"
+    )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=0,
+        help="Parallel worker processes for OA processing (0=all CPUs, 1=sequential)",
+    )
+    parser.add_argument(
+        "--greenspace",
+        type=Path,
+        default=None,
+        help="Greenspace/water parquet for boundary trimming (optional)",
+    )
+    args = parser.parse_args()
+
+    fragments_cache = args.output / "fragments_cache.parquet"
+    # Phase 3 depends only on these inputs; greenspace is applied later (Phase 4),
+    # so a greenspace change must not invalidate the fragment cache.
+    fragment_inputs = [args.uprn, args.arcgis, args.oa_boundaries, args.inspire]
+    # --limit yields a partial fragment set; never read or write the shared cache.
+    use_cache = args.limit == 0
+
+    if use_cache and fragments_cache_is_fresh(fragments_cache, fragment_inputs):
+        print("=" * 60)
+        print("Phase 3 cache hit — loading fragments (skipping Phases 1-3)")
+        print("=" * 60)
+        all_fragments = load_fragments(fragments_cache)
+        print(
+            f"  Loaded {len(all_fragments):,} cached fragments from {fragments_cache}"
+        )
+    else:
+        all_fragments = build_fragments(args)
+        if use_cache:
+            # Persist the expensive Phase-3 output before the cheap-but-fragile
+            # merge/write so any failure there resumes in seconds, not ~10 hours.
+            save_fragments(fragments_cache, all_fragments)
+            print(f"  Cached {len(all_fragments):,} fragments to {fragments_cache}")
+
+    # Free Phase-1-3 intermediates (build_fragments' locals) back to the OS.
     release_memory()
 
     # Phase 4: Merge and write
@@ -139,6 +310,12 @@ def main() -> None:
 
     file_count = write_district_geojson(merged, args.output)
     print(f"\n  Wrote {file_count} district GeoJSON files to {args.output / 'units'}")
+
+    # The cache exists only to survive a crash between Phase 3 and a clean write.
+    # Now that the output is complete, drop it so a later input change can never
+    # be served from a stale cache.
+    if use_cache:
+        fragments_cache.unlink(missing_ok=True)
     print("Done!")
 
 

pipeline/transform/postcode_boundaries/fragments_cache.py

@@ -0,0 +1,79 @@
+"""Persist Phase-3 output (the per-postcode fragments) so a crash in the later
+merge/write phases can resume in seconds instead of re-running the ~10-hour OA
+loop.
+
+Phase 3 turns OA boundaries + INSPIRE parcels + UPRN points into ~1.8M
+``(postcode, geometry)`` fragments held only in memory. Everything after it
+(merge, simplify, GeoJSON write) is cheap but failure-prone -- a single
+degenerate postcode used to abort the whole run *after* those 10 hours. Caching
+the fragments to disk decouples the expensive computation from the fragile
+output stage.
+
+Fragments are stored as one parquet file with two columns: ``postcode``
+(string) and ``wkb`` (binary Shapely WKB). Writes are atomic (temp file +
+``os.replace``) so an interrupted write never leaves a cache that passes the
+freshness check. The cache is validated against its upstream inputs by mtime: if
+any input is newer than the cache it is treated as stale and ignored, mirroring
+make's own freshness logic.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import numpy as np
+import polars as pl
+import shapely
+from shapely.geometry.base import BaseGeometry
+
+Fragment = tuple[str, BaseGeometry]
+
+
+def _tmp_path(cache_path: Path) -> Path:
+    return cache_path.parent / (cache_path.name + ".tmp")
+
+
+def fragments_cache_is_fresh(
+    cache_path: Path, inputs: list[Path | None]
+) -> bool:
+    """True if ``cache_path`` exists and is newer than every input that exists.
+
+    A missing input is ignored (it cannot have changed the cache); a ``None``
+    input is skipped. Any existing input newer than the cache marks it stale.
+    """
+    if not cache_path.exists():
+        return False
+    cache_mtime = cache_path.stat().st_mtime
+    for inp in inputs:
+        if inp is None:
+            continue
+        path = Path(inp)
+        if path.exists() and path.stat().st_mtime > cache_mtime:
+            return False
+    return True
+
+
+def save_fragments(cache_path: Path, fragments: list[Fragment]) -> None:
+    """Atomically write ``(postcode, geometry)`` fragments to a parquet cache."""
+    postcodes = [pc for pc, _ in fragments]
+    geoms = np.array([geom for _, geom in fragments], dtype=object)
+    wkb = shapely.to_wkb(geoms)
+
+    frame = pl.DataFrame(
+        {"postcode": postcodes, "wkb": list(wkb)},
+        schema={"postcode": pl.Utf8, "wkb": pl.Binary},
+    )
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = _tmp_path(cache_path)
+    frame.write_parquet(tmp, compression="zstd")
+    os.replace(tmp, cache_path)
+
+
+def load_fragments(cache_path: Path) -> list[Fragment]:
+    """Read fragments written by :func:`save_fragments` back into memory."""
+    frame = pl.read_parquet(cache_path)
+    postcodes = frame["postcode"].to_list()
+    geoms = shapely.from_wkb(frame["wkb"].to_list())
+    return list(zip(postcodes, geoms))

pipeline/transform/postcode_boundaries/geometry.py

@@ -0,0 +1,123 @@
+"""Robust GEOS overlay helpers.
+
+Overlay operations (union, difference, intersection) can raise a
+``GEOSException`` — most often ``TopologyException: side location conflict``,
+``Ring edge missing``, or ``found non-noded intersection`` — on geometries that
+contain near-coincident or near-degenerate edges, or that are individually
+invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG
+engine, handed a grid size, nodes every edge onto that grid and finishes where
+the full-precision overlay cannot.
+
+Getting the fallback to actually survive takes three rules, each of which we
+learned the hard way from a crash:
+
+1. **Never precision-reduce with the default mode.** ``set_precision``'s default
+   ``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that
+   re-raises the very ``side location conflict`` we are trying to escape. We push
+   the grid into the overlay via the ``grid_size`` argument instead — where
+   OverlayNG nodes robustly — and only ever call ``set_precision`` in
+   ``pointwise`` mode (pure coordinate rounding, which cannot raise).
+2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties,
+   pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape.
+3. **Keep only polygonal parts.** ``make_valid`` of a spiky polygon routinely
+   returns a *mixed-dimension* ``GeometryCollection`` (a polygon plus a dangling
+   line), and OverlayNG rejects mixed-dimension input with
+   ``IllegalArgumentException: Overlay input is mixed-dimension``. Every input
+   here represents an area, so the line/point debris is meaningless noise and is
+   dropped before the overlay.
+
+The default fallback grid is 0.1 mm in the metre-based working CRS
+(EPSG:27700), far below survey resolution and the ``MIN_GEOM_AREA`` threshold
+downstream, so it cannot crush a postcode-scale shape. Callers operating in a
+different CRS (e.g. the WGS84 output stage, where the same 1e-4 grid would be
+~11 m) pass a ``grid`` matched to that CRS. The whole fallback runs ONLY after
+the exact operation has already failed, so normal output is bit-for-bit
+unchanged.
+"""
+
+import shapely
+from shapely import GEOSException, make_valid, set_precision
+from shapely.geometry import Polygon
+from shapely.ops import unary_union
+
+# 0.1 mm in metres — well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
+_SNAP_GRID = 1e-4
+
+_EMPTY = Polygon()
+
+
+def _poly_valid(geom, grid):
+    """Return an OGC-valid, polygonal-only version of ``geom``.
+
+    Repairs invalidity with ``make_valid`` and discards any non-polygonal debris
+    (dangling lines/points) it leaves behind, since OverlayNG rejects
+    mixed-dimension ``GeometryCollection`` input. The result is always a
+    ``Polygon``/``MultiPolygon`` (possibly empty), safe to feed to a
+    fixed-precision overlay.
+    """
+    g = geom if geom.is_valid else make_valid(geom)
+    if g.geom_type in ("Polygon", "MultiPolygon"):
+        return g
+    if g.geom_type == "GeometryCollection":
+        polys = [
+            p
+            for p in g.geoms
+            if p.geom_type in ("Polygon", "MultiPolygon") and not p.is_empty
+        ]
+        if not polys:
+            return _EMPTY
+        if len(polys) == 1:
+            return polys[0]
+        # Dissolve on the grid (robust) rather than building a possibly-invalid
+        # MultiPolygon of overlapping parts.
+        return shapely.union_all(polys, grid_size=grid)
+    return _EMPTY  # line / point only
+
+
+def _snap_poly_valid(geom, grid):
+    """Coordinate-round onto ``grid`` (``pointwise`` never raises), then clean."""
+    return _poly_valid(set_precision(geom, grid, mode="pointwise"), grid)
+
+
+def safe_union(geoms, grid=_SNAP_GRID):
+    """``unary_union`` that survives GEOS robustness failures."""
+    try:
+        return unary_union(geoms)
+    except GEOSException:
+        pass
+    cleaned = [_poly_valid(g, grid) for g in geoms]
+    try:
+        return shapely.union_all(cleaned, grid_size=grid)
+    except GEOSException:
+        snapped = [_snap_poly_valid(g, grid) for g in geoms]
+        return shapely.union_all(snapped, grid_size=grid)
+
+
+def safe_difference(a, b, grid=_SNAP_GRID):
+    """``a.difference(b)`` that survives GEOS robustness failures."""
+    try:
+        return a.difference(b)
+    except GEOSException:
+        pass
+    a2, b2 = _poly_valid(a, grid), _poly_valid(b, grid)
+    try:
+        return shapely.difference(a2, b2, grid_size=grid)
+    except GEOSException:
+        return shapely.difference(
+            _snap_poly_valid(a2, grid), _snap_poly_valid(b2, grid), grid_size=grid
+        )
+
+
+def safe_intersection(a, b, grid=_SNAP_GRID):
+    """``a.intersection(b)`` that survives GEOS robustness failures."""
+    try:
+        return a.intersection(b)
+    except GEOSException:
+        pass
+    a2, b2 = _poly_valid(a, grid), _poly_valid(b, grid)
+    try:
+        return shapely.intersection(a2, b2, grid_size=grid)
+    except GEOSException:
+        return shapely.intersection(
+            _snap_poly_valid(a2, grid), _snap_poly_valid(b2, grid), grid_size=grid
+        )

pipeline/transform/postcode_boundaries/greenspace.py

@@ -5,9 +5,10 @@ from pathlib import Path
 import polars as pl
 from shapely import wkb
 from shapely.geometry import MultiPolygon, Polygon
-from shapely.ops import unary_union
 from shapely.strtree import STRtree
 
+from .geometry import safe_difference, safe_union
+
 
 def load_greenspace(path: Path) -> tuple[STRtree, list]:
     """Load greenspace parquet and build an STRtree spatial index.
@@ -51,8 +52,8 @@ def subtract_greenspace(
     if not intersecting:
         return postcode_geom
 
-    green_union = unary_union(intersecting)
-    result = postcode_geom.difference(green_union)
+    green_union = safe_union(intersecting)
+    result = safe_difference(postcode_geom, green_union)
 
     if result.is_empty:
         return postcode_geom

pipeline/transform/postcode_boundaries/inspire.py

@@ -112,37 +112,113 @@ def load_inspire(
     return bboxes, offsets, coords_mmap
 
 
-def get_inspire_candidates(
-    oa_bounds: tuple[float, float, float, float],
+# Grid cell size (m) for the parcel spatial index. The median parcel is ~25 m
+# and the 99th percentile ~540 m, so almost every parcel fits inside a single
+# 1 km cell; the ~0.4% larger than a cell go to an overflow list tested on every
+# query.
+_GRID_CELL_SIZE = 1000.0
+
+
+class InspireIndex:
+    """Uniform-grid spatial index over INSPIRE parcel bounding boxes.
+
+    The per-OA candidate lookup used to linear-scan all ~24M bboxes (O(N) per
+    OA, ~4 h total over the country). This indexes parcels by grid cell so each
+    lookup is O(cells_spanned + candidates). Parcels no larger than one cell are
+    bucketed by their bbox min-corner cell in a CSR layout (parcel indices sorted
+    by cell id, located with ``searchsorted``); the few parcels larger than a
+    cell are kept in an overflow array tested directly on every query. An exact
+    bbox test then runs on the gathered subset and the result is sorted, so the
+    candidate set -- and its order -- is byte-for-byte identical to the old scan.
+    """
+
+    def __init__(
+        self,
         bboxes: np.ndarray,
         offsets: np.ndarray,
         coords_mmap: np.memmap,
-) -> list[Polygon]:
-    """Get INSPIRE polygons overlapping an OA via bbox pre-filter.
+        cell_size: float = _GRID_CELL_SIZE,
+    ) -> None:
+        self._bboxes = bboxes
+        self._offsets = offsets
+        self._coords = coords_mmap
+        self._cell_size = cell_size
+        self._origin_x = float(bboxes[:, 0].min())
+        self._origin_y = float(bboxes[:, 1].min())
+        # Flattened cell id is ``cx * _ny + cy``; +2 leaves a guard row so the
+        # query's one-cell low-edge widening can never collide with cx-1.
+        self._ny = int((bboxes[:, 1].max() - self._origin_y) // cell_size) + 2
+
+        width = bboxes[:, 2] - bboxes[:, 0]
+        height = bboxes[:, 3] - bboxes[:, 1]
+        small = np.where((width <= cell_size) & (height <= cell_size))[0]
+        self._oversized = np.where((width > cell_size) | (height > cell_size))[0]
+        self._oversized_bb = bboxes[self._oversized]
+
+        cx = ((bboxes[small, 0] - self._origin_x) // cell_size).astype(np.int64)
+        cy = ((bboxes[small, 1] - self._origin_y) // cell_size).astype(np.int64)
+        cell_id = cx * self._ny + cy
+        order = np.argsort(cell_id, kind="stable")
+        self._sorted_cells = cell_id[order]
+        self._cell_parcels = small[order]
+
+    def candidate_indices(self, oa_bounds: tuple[float, float, float, float]) -> np.ndarray:
+        """Parcel indices whose bbox overlaps ``oa_bounds`` (ascending order)."""
+        min_e, min_n, max_e, max_n = oa_bounds
+        cs = self._cell_size
+        # A small parcel (<= one cell) overlapping the OA has its min-corner no
+        # more than one cell below/left of the OA bbox, so widen the low edges by
+        # a cell. This keeps the lookup free of false negatives.
+        gx0 = int((min_e - cs - self._origin_x) // cs)
+        gx1 = int((max_e - self._origin_x) // cs)
+        gy_lo = int((min_n - cs - self._origin_y) // cs)
+        gy_hi = int((max_n - self._origin_y) // cs)
+
+        parts = []
+        ob = self._oversized_bb
+        if len(ob):
+            mo = (
+                (ob[:, 2] >= min_e)
+                & (ob[:, 0] <= max_e)
+                & (ob[:, 3] >= min_n)
+                & (ob[:, 1] <= max_n)
+            )
+            if mo.any():
+                parts.append(self._oversized[mo])
+
+        for gx in range(gx0, gx1 + 1):
+            base = gx * self._ny
+            lo = np.searchsorted(self._sorted_cells, base + gy_lo, "left")
+            hi = np.searchsorted(self._sorted_cells, base + gy_hi, "right")
+            if hi > lo:
+                parts.append(self._cell_parcels[lo:hi])
+
+        if not parts:
+            return np.empty(0, dtype=np.int64)
+        cand = np.concatenate(parts)
+        cb = self._bboxes[cand]
+        mask = (
+            (cb[:, 2] >= min_e)
+            & (cb[:, 0] <= max_e)
+            & (cb[:, 3] >= min_n)
+            & (cb[:, 1] <= max_n)
+        )
+        # Sort so the candidate order matches the old full np.where scan exactly.
+        return np.sort(cand[mask])
+
+    def candidates(
+        self, oa_bounds: tuple[float, float, float, float]
+    ) -> list[Polygon]:
+        """INSPIRE polygons overlapping an OA, built from the mmap on demand.
 
         Builds Shapely objects only for matches (typically 10-500 per OA).
-    Reads coordinate data on-demand from memory-mapped file.
         """
-    min_e, min_n, max_e, max_n = oa_bounds
-
-    # Vectorized bbox overlap test
-    mask = (
-        (bboxes[:, 2] >= min_e)
-        & (bboxes[:, 0] <= max_e)
-        & (bboxes[:, 3] >= min_n)
-        & (bboxes[:, 1] <= max_n)
-    )
-    idxs = np.where(mask)[0]
-    if len(idxs) == 0:
-        return []
-
-    # Build Shapely polygons only for candidates (coords from mmap)
         candidates = []
-    for i in idxs:
-        byte_offset = offsets[i, 0]
-        n_pts = offsets[i, 1]
+        for i in self.candidate_indices(oa_bounds):
+            byte_offset = self._offsets[i, 0]
+            n_pts = self._offsets[i, 1]
             float_offset = byte_offset // 8  # float64 = 8 bytes
-        coords = coords_mmap[float_offset : float_offset + n_pts * 2].reshape(-1, 2)
+            coords = self._coords[float_offset : float_offset + n_pts * 2].reshape(-1, 2)
             poly = Polygon(coords)
             if not poly.is_valid:
                 poly = make_valid(poly)
@@ -153,3 +229,13 @@ def get_inspire_candidates(
             if not poly.is_empty:
                 candidates.append(poly)
         return candidates
+
+
+def build_inspire_index(
+    bboxes: np.ndarray,
+    offsets: np.ndarray,
+    coords_mmap: np.memmap,
+    cell_size: float = _GRID_CELL_SIZE,
+) -> InspireIndex:
+    """Build the grid spatial index used for per-OA candidate retrieval."""
+    return InspireIndex(bboxes, offsets, coords_mmap, cell_size)

pipeline/transform/postcode_boundaries/loader.py

@@ -0,0 +1,105 @@
+"""Load per-district postcode boundary GeoJSONs as EPSG:27700 polygons.
+
+The postcode-boundary pipeline (:mod:`output`) writes one WGS84 GeoJSON per
+postcode district under ``units/{district}.geojson``, each feature carrying a
+``postcodes`` (full unit string, e.g. "AL1 1AG") property. Spatial transforms
+that test points against postcode geometry want those polygons back in British
+National Grid (EPSG:27700) so buffers/distances are in metres.
+
+:func:`load_postcode_polygons` reads the files, reprojects WGS84→27700, repairs
+invalid rings, and returns parallel ``(postcodes, polygons)`` arrays sorted by
+postcode so callers can use the array index as a stable postcode id -- the same
+"buffer index == postcode index" convention used by ``tree_density``.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+import shapely
+from pyproj import Transformer
+
+
+def _read_district(
+    path: Path, transformer: Transformer
+) -> tuple[np.ndarray, np.ndarray]:
+    """Return (postcodes, polygons_27700) for one district GeoJSON."""
+    with path.open() as file:
+        collection = json.load(file)
+
+    features = collection.get("features", [])
+    if not features:
+        return np.empty(0, dtype=object), np.empty(0, dtype=object)
+
+    postcodes = np.array(
+        [feature["properties"]["postcodes"] for feature in features], dtype=object
+    )
+    geom_json = np.array(
+        [json.dumps(feature["geometry"]) for feature in features], dtype=object
+    )
+    geoms = shapely.from_geojson(geom_json)
+
+    # Reproject every vertex in a single pyproj call, then rebuild the polygons.
+    coords = shapely.get_coordinates(geoms)
+    if coords.size:
+        x, y = transformer.transform(coords[:, 0], coords[:, 1])
+        geoms = shapely.set_coordinates(geoms, np.column_stack([x, y]))
+
+    invalid = ~shapely.is_valid(geoms)
+    if invalid.any():
+        geoms[invalid] = shapely.make_valid(geoms[invalid])
+
+    return postcodes, geoms
+
+
+def load_postcode_polygons(
+    units_dir: Path, max_postcodes: int | None = None
+) -> tuple[np.ndarray, np.ndarray]:
+    """Load all postcode polygons under ``units_dir`` reprojected to EPSG:27700.
+
+    Returns ``(postcodes, polygons)`` parallel object arrays sorted by postcode.
+    ``max_postcodes`` (testing) keeps only the lexicographically-first N
+    postcodes, reading just enough district files to reach the cap.
+    """
+    units_dir = Path(units_dir)
+    files = sorted(units_dir.glob("*.geojson"))
+    if not files:
+        raise FileNotFoundError(f"No postcode-boundary GeoJSONs found in {units_dir}")
+
+    transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
+    postcode_chunks: list[np.ndarray] = []
+    geom_chunks: list[np.ndarray] = []
+    total = 0
+    for path in files:
+        postcodes, geoms = _read_district(path, transformer)
+        if len(postcodes) == 0:
+            continue
+        postcode_chunks.append(postcodes)
+        geom_chunks.append(geoms)
+        total += len(postcodes)
+        if max_postcodes is not None and total >= max_postcodes:
+            break
+
+    if not postcode_chunks:
+        raise ValueError(f"No postcode features found in {units_dir}")
+
+    postcodes = np.concatenate(postcode_chunks)
+    geoms = np.concatenate(geom_chunks)
+
+    # Stable postcode order makes "index == postcode id" deterministic; dedupe
+    # defensively (a postcode lives in exactly one district file).
+    order = np.argsort(postcodes, kind="stable")
+    postcodes = postcodes[order]
+    geoms = geoms[order]
+    _, first = np.unique(postcodes, return_index=True)
+    postcodes = postcodes[first]
+    geoms = geoms[first]
+
+    if max_postcodes is not None and len(postcodes) > max_postcodes:
+        postcodes = postcodes[:max_postcodes]
+        geoms = geoms[:max_postcodes]
+
+    print(f"Loaded {len(postcodes):,} postcode polygons from {units_dir}")
+    return postcodes, geoms

pipeline/transform/postcode_boundaries/output.py

@@ -1,13 +1,18 @@
 import json
+import shutil
 from collections import defaultdict
 from pathlib import Path
 
+import numpy as np
 from pyproj import Transformer
-from shapely import make_valid
-from shapely.geometry import MultiPolygon, Polygon
-from shapely.ops import unary_union
+from shapely import STRtree, make_valid, set_precision
+from shapely.errors import GEOSException
+from shapely.geometry import MultiPolygon, Polygon, box, mapping, shape
+from shapely.ops import transform as transform_geometry
 from tqdm import tqdm
 
+from .geometry import safe_difference, safe_union
+
 _to_wgs84 = None
 
 
@@ -18,65 +23,171 @@ def _get_to_wgs84():
     return _to_wgs84
 
 
+def _largest_polygonal(geom) -> Polygon | None:
+    if geom is None or geom.is_empty:
+        return None
+    if not geom.is_valid:
+        geom = make_valid(geom)
+    if geom.geom_type == "Polygon":
+        return geom
+    if geom.geom_type == "MultiPolygon":
+        return max(geom.geoms, key=lambda g: g.area)
+    if geom.geom_type == "GeometryCollection":
+        polygons = [
+            polygon
+            for part in geom.geoms
+            if (polygon := _largest_polygonal(part)) is not None
+        ]
+        if polygons:
+            return max(polygons, key=lambda g: g.area)
+    return None
+
+
+# Output coordinate grid (~0.11 m at UK latitudes). Polygons whose extent is
+# below this in any direction snap to empty during serialization.
+_OUTPUT_PRECISION_DEG = 0.000001
+# Minimal BNG buffer used to rescue sub-grid slivers into a representable
+# footprint. A near-zero-area Voronoi/INSPIRE spike (e.g. three almost-collinear
+# vertices) would otherwise vanish at output precision; since every *active*
+# postcode must keep a boundary (validate_outputs enforces this with zero
+# tolerance), we fatten it just enough to survive snapping rather than drop it.
+_MIN_FOOTPRINT_BUFFER_M = 0.5
+
+
+def _snap_to_wgs84_geojson(geom_bng: Polygon | MultiPolygon) -> dict | None:
+    """Transform a BNG polygon to WGS84, snap to output precision, validate.
+
+    Validates the *serialized* GeoJSON dict (via a ``shape()`` round-trip), not
+    just the intermediate Shapely object: coordinate snapping during
+    serialization can otherwise leave a self-intersecting ring that only shows up
+    once the feature is read back from disk. Returns ``None`` if the geometry
+    collapses to empty (a sub-grid sliver).
+    """
+    transformer = _get_to_wgs84()
+    wgs84 = transform_geometry(transformer.transform, geom_bng)
+    try:
+        wgs84 = set_precision(wgs84, _OUTPUT_PRECISION_DEG, mode="valid_output")
+    except GEOSException:
+        # Precision snapping can fail on pathological geometries; fall back to a
+        # plain validity repair without coordinate snapping.
+        wgs84 = make_valid(wgs84)
+    wgs84 = _largest_polygonal(wgs84)
+    if wgs84 is None:
+        return None
+
+    geojson_dict = mapping(wgs84)
+
+    # The geometry that actually reaches disk is the GeoJSON dict, so validate
+    # *that* (not the pre-serialization object) and repair if needed.
+    round_trip = shape(geojson_dict)
+    if round_trip.is_empty or not round_trip.is_valid:
+        round_trip = _largest_polygonal(make_valid(round_trip))
+        if round_trip is None or round_trip.is_empty:
+            return None
+        geojson_dict = mapping(round_trip)
+
+    return geojson_dict
+
+
+def _rescue_footprint(geom_bng) -> dict | None:
+    """Fatten a degenerate BNG geometry into a representable footprint and snap."""
+    footprint = _largest_polygonal(geom_bng.buffer(_MIN_FOOTPRINT_BUFFER_M))
+    if footprint is None:
+        return None
+    return _snap_to_wgs84_geojson(footprint)
+
+
 def to_wgs84_geojson(
     geom: Polygon | MultiPolygon, tolerance: float = 1.0
 ) -> dict | None:
-    """Simplify geometry in BNG, convert to WGS84, return GeoJSON dict."""
-    if geom.is_empty:
+    """Simplify geometry in BNG, convert to WGS84, return a valid GeoJSON dict.
+
+    A few thousand postcodes reduce to a sub-grid sliver that snaps to empty at
+    output precision. Dropping them would leave an active postcode with no
+    boundary (validate_outputs rejects that with zero tolerance), so instead they
+    are fattened into a minimal footprint at the right location: first by buffering
+    the (often elongated) sliver itself, then -- for fully-degenerate input -- a
+    small disc around ``representative_point()``, which lies inside any non-empty
+    geometry. ``None`` is returned only for a genuinely empty input.
+    """
+    if geom is None or geom.is_empty:
         return None
 
-    simplified = geom.simplify(tolerance, preserve_topology=True)
-    if simplified.is_empty:
+    cleaned = _largest_polygonal(geom)
+    if cleaned is not None:
+        simplified = _largest_polygonal(
+            cleaned.simplify(tolerance, preserve_topology=True)
+        )
+        if simplified is None:
+            simplified = cleaned
+        # Normal path; if snapping erases a thin sliver, fatten its real shape.
+        result = _snap_to_wgs84_geojson(simplified)
+        if result is None:
+            result = _rescue_footprint(simplified)
+        if result is not None:
+            return result
+
+    # Universal fallback for input too degenerate to clean or fatten in place.
+    return _rescue_footprint(geom.representative_point())
+
+
+def to_wgs84_geojson_multi(
+    geom: Polygon | MultiPolygon, tolerance: float = 1.0
+) -> dict | None:
+    """Convert a (possibly multi-part) postcode geometry to a GeoJSON dict,
+    preserving every part. Each part is simplified/snapped/rescued independently
+    via :func:`to_wgs84_geojson`; the result is a ``Polygon`` for a single part or
+    a ``MultiPolygon`` for several. ``None`` only if every part is degenerate.
+    """
+    parts = list(geom.geoms) if geom.geom_type == "MultiPolygon" else [geom]
+    part_dicts = [d for part in parts if (d := to_wgs84_geojson(part, tolerance))]
+    if not part_dicts:
         return None
-
-    transformer = _get_to_wgs84()
-
-    def transform_ring(coords):
-        xs, ys = zip(*coords)
-        lons, lats = transformer.transform(list(xs), list(ys))
-        return [(round(lon, 6), round(lat, 6)) for lon, lat in zip(lons, lats)]
-
-    def transform_polygon(poly):
-        exterior = transform_ring(poly.exterior.coords)
-        holes = [transform_ring(h.coords) for h in poly.interiors]
-        return [exterior] + holes
-
-    # Force single Polygon — postcodes are contiguous delivery routes
-    if simplified.geom_type == "MultiPolygon":
-        simplified = max(simplified.geoms, key=lambda g: g.area)
-    elif simplified.geom_type == "GeometryCollection":
-        polys = [
-            g for g in simplified.geoms if g.geom_type in ("Polygon", "MultiPolygon")
-        ]
-        if not polys:
-            return None
-        simplified = max(polys, key=lambda g: g.area)
-        if simplified.geom_type == "MultiPolygon":
-            simplified = max(simplified.geoms, key=lambda g: g.area)
-
-    if simplified.geom_type != "Polygon" or simplified.is_empty:
-        return None
-
+    if len(part_dicts) == 1:
+        return part_dicts[0]
     return {
-        "type": "Polygon",
-        "coordinates": transform_polygon(simplified),
+        "type": "MultiPolygon",
+        "coordinates": [pd["coordinates"] for pd in part_dicts],
     }
 
 
+# Interior holes from the INSPIRE+Voronoi+make_valid chain are small artifacts and
+# get filled. A hole at least this large is likely a genuinely enclosed postcode
+# (kept, so we never solidify over a neighbour); the de-overlap pass is the real
+# guarantee, this is defence-in-depth.
+_MAX_ARTIFACT_HOLE_AREA = 1000.0
+
+
+def _fill_small_holes(poly: Polygon) -> Polygon:
+    kept = [r for r in poly.interiors if Polygon(r).area >= _MAX_ARTIFACT_HOLE_AREA]
+    return Polygon(poly.exterior, kept)
+
+
 def _fill_holes(geom):
-    """Remove all interior rings (holes) from a polygon or multipolygon."""
+    """Fill small artifact interior rings; keep large (real-enclosed) holes."""
     if geom.geom_type == "Polygon":
-        return Polygon(geom.exterior)
+        return _fill_small_holes(geom)
     elif geom.geom_type == "MultiPolygon":
-        return MultiPolygon([Polygon(p.exterior) for p in geom.geoms])
+        return MultiPolygon([_fill_small_holes(p) for p in geom.geoms])
     return geom
 
 
-def _largest_polygon(geom):
-    """Extract the largest polygon from a MultiPolygon."""
-    if geom.geom_type == "MultiPolygon":
-        return max(geom.geoms, key=lambda g: g.area)
+# A postcode genuinely split across an OA seam (by a railway, river, or main road
+# wider than the merge buffer) arrives here as a MultiPolygon. Keeping only the
+# largest part used to discard the rest, leaving ~1.8% of merged area as uncovered
+# gaps (often 3000-5000 m² building blocks). Keep every part at least this big;
+# smaller detached bits are Voronoi/clipping noise and are still dropped.
+_MIN_DETACHED_PART_AREA = 100.0
+
+
+def _keep_polygon_parts(geom):
+    """Keep all MultiPolygon parts >= _MIN_DETACHED_PART_AREA (largest if none)."""
+    if geom.geom_type != "MultiPolygon":
         return geom
+    parts = [g for g in geom.geoms if g.area >= _MIN_DETACHED_PART_AREA]
+    if not parts:
+        parts = [max(geom.geoms, key=lambda g: g.area)]
+    return parts[0] if len(parts) == 1 else MultiPolygon(parts)
 
 
 def merge_fragments(
@@ -97,19 +208,24 @@ def merge_fragments(
 
     merged = {}
     for pc, parts in by_postcode.items():
-        combined = unary_union(parts)
+        combined = safe_union(parts)
         if combined.is_empty:
             continue
         if not combined.is_valid:
             combined = make_valid(combined)
-        # Close tiny gaps between adjacent OA boundary edges (float mismatches)
+        # Close tiny gaps between adjacent OA boundary edges (float mismatches).
+        # The closing can erode a tiny MultiPolygon (e.g. a postcode with only a
+        # sliver fragment) to nothing, which would leave the postcode with no
+        # geometry at all — keep the un-closed shape if that happens.
         if combined.geom_type == "MultiPolygon":
-            combined = combined.buffer(5.0).buffer(-5.0)
-            if not combined.is_valid:
-                combined = make_valid(combined)
-        # Postcodes are contiguous delivery routes — keep only the largest
-        # polygon; small detached fragments are algorithm artifacts
-        combined = _largest_polygon(combined)
+            closed = combined.buffer(5.0).buffer(-5.0)
+            if not closed.is_valid:
+                closed = make_valid(closed)
+            if not closed.is_empty:
+                combined = closed
+        # Keep the postcode whole: the largest part plus any other substantial
+        # part (a genuine railway/river split), dropping only tiny noise slivers.
+        combined = _keep_polygon_parts(combined)
         # Remove artifact interior holes from INSPIRE+Voronoi+make_valid chain
         combined = _fill_holes(combined)
         # Subtract parks/water if provided
@@ -118,8 +234,12 @@ def merge_fragments(
 
             pre_green = combined
             combined = subtract_greenspace(combined, greenspace_tree, greenspace_geoms)
-            combined = _largest_polygon(combined)
-            combined = _fill_holes(combined)
+            combined = _keep_polygon_parts(combined)
+            # Do NOT _fill_holes here: interior holes carved by the greenspace
+            # subtraction (lakes, enclosed parks) are intentional, not artifacts.
+            # Filling them would re-add the removed area and negate the
+            # subtraction. Artifact holes from the INSPIRE+Voronoi+make_valid
+            # chain were already removed by the _fill_holes above (pre-subtraction).
             # Revert if subtraction + fragment selection lost >90% of area
             if pre_green.area > 0 and combined.area / pre_green.area < 0.1:
                 combined = pre_green
@@ -127,15 +247,218 @@ def merge_fragments(
     return merged
 
 
+def _polygonal(geom):
+    """Return only the polygonal part(s) of a geometry, or None if none remain."""
+    if geom is None or geom.is_empty:
+        return None
+    if geom.geom_type in ("Polygon", "MultiPolygon"):
+        return geom
+    if geom.geom_type == "GeometryCollection":
+        polys = [
+            g
+            for g in geom.geoms
+            if g.geom_type in ("Polygon", "MultiPolygon") and not g.is_empty
+        ]
+        if not polys:
+            return None
+        # Both callers run on WGS84-degree output geometry, so the robustness
+        # fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre
+        # default — a coarse metre grid would obliterate a degree-scale shape.
+        merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG)
+        return merged if not merged.is_empty else None
+    return None
+
+
+def _resolve_overlaps(
+    items: list[tuple[str, Polygon | MultiPolygon]],
+) -> list[tuple[str, Polygon | MultiPolygon]]:
+    """Make the postcode polygons a partition: no two cover the same ground.
+
+    Overlap appears at OA seams (the 5m merge buffer expands each postcode
+    independently), from simplifying each postcode on its own, and as genuine
+    containment (a postcode fully enclosed by another). Each postcode is trimmed
+    by the union of its higher-priority overlapping neighbours, where **priority =
+    ascending area**: a smaller postcode wins contested ground. That single rule
+    handles both cases correctly — an enclosed postcode is always smaller than its
+    container, so it keeps its area while the container gets a hole (a `overlaps`
+    query alone would miss containment entirely). Run last, on the final output
+    geometries, so nothing re-introduces overlap afterwards. A postcode that would
+    be emptied keeps its original geometry, so an active postcode is never dropped.
+    """
+    geoms = [g for _, g in items]
+    n = len(geoms)
+    if n < 2:
+        return items
+
+    # rank[i]: 0 = highest priority (smallest area). Postcode string breaks ties
+    # for determinism.
+    rank = {
+        idx: r
+        for r, idx in enumerate(
+            sorted(range(n), key=lambda i: (geoms[i].area, items[i][0]))
+        )
+    }
+
+    tree = STRtree(geoms)
+    arr = np.array(geoms, dtype=object)
+    pairs: set[tuple[int, int]] = set()
+    # "overlaps" gives partial overlaps; "contains" gives containment (which
+    # "overlaps" excludes) — together they cover every 2-D overlap without the
+    # edge-touch explosion a plain "intersects" query would add.
+    for predicate in ("overlaps", "contains"):
+        qsrc, qtgt = tree.query(arr, predicate=predicate)
+        for s, t in zip(qsrc.tolist(), qtgt.tolist()):
+            if s != t:
+                pairs.add((s, t) if s < t else (t, s))
+
+    # For each loser (lower priority) the higher-priority neighbours to subtract.
+    higher: dict[int, list[int]] = defaultdict(list)
+    for a, b in pairs:
+        winner, loser = (a, b) if rank[a] < rank[b] else (b, a)
+        higher[loser].append(winner)
+
+    out = list(geoms)
+    # Process losers from highest priority down, so every subtracted neighbour is
+    # already finalised.
+    for i in sorted(higher, key=lambda idx: rank[idx]):
+        # These geometries are WGS84 degrees already snapped to output precision,
+        # so the robustness fallback snaps on the same 1e-6° grid (~0.11 m) rather
+        # than geometry.py's metre-CRS default. A raw difference can still raise a
+        # "side location conflict" on near-coincident OA-seam edges; the
+        # fixed-precision overlay noded them away.
+        cut = safe_union([out[j] for j in higher[i]], grid=_OUTPUT_PRECISION_DEG)
+        trimmed = safe_difference(out[i], cut, grid=_OUTPUT_PRECISION_DEG)
+        if not trimmed.is_valid:
+            trimmed = make_valid(trimmed)
+        # Keep all polygonal parts: these geometries are in WGS84 degrees, so an
+        # area threshold here would wrongly drop everything but the largest part
+        # and re-open the very gaps the seam fix closed.
+        trimmed = _polygonal(trimmed)
+        if trimmed is not None and not trimmed.is_empty:
+            out[i] = trimmed
+    return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
+
+
+def _round_coords(coords, ndigits=6):
+    if coords and isinstance(coords[0], (int, float)):
+        return [round(coords[0], ndigits), round(coords[1], ndigits)]
+    return [_round_coords(c, ndigits) for c in coords]
+
+
+def _snap_polygonal(geom, grid):
+    """Re-node ``geom`` onto ``grid`` (``valid_output``) → polygonal-only, or None.
+
+    ``set_precision`` ``valid_output`` runs an OverlayNG noding pass that places
+    every vertex on a multiple of ``grid`` *and* fixes the topology, so a plain
+    coordinate round of the result is exact (no two distinct vertices can land in
+    the same cell). Falls back to ``make_valid`` if precision-reduction raises.
+    """
+    try:
+        snapped = set_precision(geom, grid, mode="valid_output")
+    except GEOSException:
+        snapped = make_valid(geom)
+    return _polygonal(snapped if snapped.is_valid else make_valid(snapped))
+
+
+# A square this many output-grid cells on a side, used as the last-resort
+# footprint when snapping erases a sub-grid sliver. ~10 cells (≈0.7–1.1 m at UK
+# latitudes) is invisible at map scale yet survives the 1e-6° snap as a valid,
+# 4-corner ring.
+_FOOTPRINT_GRID_CELLS = 5
+
+
+def _grid_footprint(geom):
+    """A tiny grid-aligned square at ``geom``'s representative point, snapped valid.
+
+    Last line of defence so an *active* postcode never vanishes: the de-overlap
+    pass can shave a small (e.g. co-located, non-geographic) postcode down to a
+    sub-grid sliver that disappears when snapped to output precision. Rather than
+    drop it, place a minimal valid footprint at its location. The tiny overlap
+    this re-creates with the neighbour that trimmed it is harmless — the output
+    partition is best-effort, a missing boundary is a hard validation failure.
+    """
+    try:
+        point = geom.representative_point()
+    except GEOSException:
+        return None
+    half = _OUTPUT_PRECISION_DEG * _FOOTPRINT_GRID_CELLS
+    square = box(point.x - half, point.y - half, point.x + half, point.y + half)
+    return _snap_polygonal(square, _OUTPUT_PRECISION_DEG)
+
+
+def _geojson_geometry(geom) -> dict | None:
+    """Serialize a WGS84 polygon/multipolygon to a *valid* 6dp GeoJSON dict, or None.
+
+    The coordinates are snapped onto the 1e-6° output grid with a re-noding pass
+    BEFORE the 6dp rounding, not by the round alone. ``_resolve_overlaps`` leaves
+    thin overlap-sliver triangles with full-precision (off-grid) vertices at OA
+    seams; a bare round-to-6dp collapses those into degenerate rings (GEOS "Too
+    few points") and pinches rings into self-intersections that pass the
+    pre-rounding validity check but fail once the feature is read back from disk.
+    Snapping with ``valid_output`` nodes the geometry onto the grid so the round
+    that follows lands on exact 1e-6 multiples and cannot reintroduce invalidity.
+
+    ``None`` only for a genuinely empty/degenerate-with-no-location input; a
+    non-empty geometry that snaps to a sub-grid sliver is rescued into a minimal
+    grid footprint rather than dropped (an active postcode must keep a boundary).
+    """
+    geom = _polygonal(geom if geom.is_valid else make_valid(geom))
+    if geom is None or geom.is_empty:
+        return None
+    snapped = _snap_polygonal(geom, _OUTPUT_PRECISION_DEG)
+    if snapped is None or snapped.is_empty:
+        snapped = _grid_footprint(geom)
+        if snapped is None or snapped.is_empty:
+            return None
+    gj = mapping(snapped)
+    out = {"type": gj["type"], "coordinates": _round_coords(gj["coordinates"])}
+    # Defence-in-depth: re-validate the dict that actually reaches disk. Snapping
+    # makes the round exact, so this should already hold; repair once more on the
+    # grid if a pathological vertex still pinches a ring.
+    if not shape(out).is_valid:
+        snapped = _snap_polygonal(shape(out), _OUTPUT_PRECISION_DEG)
+        if snapped is None or snapped.is_empty:
+            return None
+        gj = mapping(snapped)
+        out = {"type": gj["type"], "coordinates": _round_coords(gj["coordinates"])}
+    return out
+
+
 def write_district_geojson(
     postcodes: dict[str, Polygon | MultiPolygon], output_dir: Path
 ) -> int:
-    """Group postcodes by district, write GeoJSON files. Returns file count."""
+    """Group postcodes by district, write GeoJSON files. Returns file count.
+
+    Before writing, the postcode polygons are converted to their final WGS84 form
+    and made a partition (overlaps removed) so the output never has two postcodes
+    covering the same ground.
+    """
     units_dir = output_dir / "units"
-    units_dir.mkdir(parents=True, exist_ok=True)
+    tmp_units_dir = output_dir / "units.tmp"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    if tmp_units_dir.exists():
+        shutil.rmtree(tmp_units_dir)
+    tmp_units_dir.mkdir(parents=True)
+
+    skipped: list[str] = []
+
+    # Pass 1: convert every postcode to its final WGS84 geometry (simplify, snap,
+    # sliver-rescue, multi-part preserved). Sorted → deterministic de-overlap
+    # priority. to_wgs84_geojson_multi returns None only for a genuinely empty
+    # input, which is skipped and reported rather than aborting a multi-hour run.
+    converted: list[tuple[str, Polygon | MultiPolygon]] = []
+    for pc in sorted(postcodes):
+        gj = to_wgs84_geojson_multi(postcodes[pc])
+        if gj is None:
+            skipped.append(pc)
+            continue
+        converted.append((pc, shape(gj)))
+
+    # Remove overlap strips so the output is a clean partition.
+    converted = _resolve_overlaps(converted)
 
     by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
-    for pc, geom in postcodes.items():
+    for pc, geom in converted:
         parts = pc.split()
         district = parts[0] if parts else pc[:4]
         by_district[district].append((pc, geom))
@@ -146,17 +469,17 @@ def write_district_geojson(
     ):
         features = []
         for pc, geom in sorted(entries, key=lambda x: x[0]):
-            geojson_geom = to_wgs84_geojson(geom)
+            geojson_geom = _geojson_geometry(geom)
             if geojson_geom is None:
+                skipped.append(pc)
                 continue
-            mapit_code = pc.replace(" ", "")
             features.append(
                 {
                     "type": "Feature",
                     "geometry": geojson_geom,
                     "properties": {
                         "postcodes": pc,
-                        "mapit_code": mapit_code,
+                        "mapit_code": pc.replace(" ", ""),
                     },
                 }
             )
@@ -165,9 +488,20 @@ def write_district_geojson(
             continue
 
         collection = {"type": "FeatureCollection", "features": features}
-        out_path = units_dir / f"{district}.geojson"
+        out_path = tmp_units_dir / f"{district}.geojson"
         with open(out_path, "w") as f:
             json.dump(collection, f, separators=(",", ":"))
         file_count += 1
 
+    if skipped:
+        preview = ", ".join(skipped[:10])
+        suffix = " …" if len(skipped) > 10 else ""
+        print(
+            f"  Skipped {len(skipped)} postcode(s) with degenerate (sub-grid) "
+            f"geometry: {preview}{suffix}"
+        )
+
+    if units_dir.exists():
+        shutil.rmtree(units_dir)
+    tmp_units_dir.replace(units_dir)
     return file_count

pipeline/transform/postcode_boundaries/process_oa.py

@@ -3,13 +3,23 @@ from collections import Counter, defaultdict
 import numpy as np
 from scipy.spatial import cKDTree
 from shapely import STRtree, make_valid
-from shapely.geometry import MultiPolygon, Polygon
-from shapely.ops import unary_union
+from shapely.geometry import MultiPoint, MultiPolygon, Point, Polygon
 
+from .geometry import safe_difference, safe_intersection, safe_union
 from .voronoi import compute_voronoi_regions
 
 MIN_GEOM_AREA = 0.01
 
+# Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a
+# crowded multi-postcode OA — its Voronoi cell ∩ remaining collapses below
+# MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a
+# co-located postcode. Every *active* postcode must keep a boundary
+# (validate_outputs is zero-tolerance), so it gets a small disc at its true seed
+# location. ~28 m² clears MIN_GEOM_AREA and output snapping; the overlap it
+# creates with the area's winner is resolved at the output stage, where the
+# smaller postcode wins the contested ground (see output._resolve_overlaps).
+_MIN_SEED_FOOTPRINT_M = 3.0
+
 
 def process_oa(
     oa_geom: Polygon | MultiPolygon,
@@ -36,10 +46,12 @@ def process_oa(
 
     # Compute remaining area
     if claimed:
-        all_claimed = unary_union(list(claimed.values()))
+        all_claimed = safe_union(list(claimed.values()))
         all_claimed = _clean_polygonal(all_claimed)
         remaining = (
-            valid_oa.difference(all_claimed) if all_claimed is not None else valid_oa
+            safe_difference(valid_oa, all_claimed)
+            if all_claimed is not None
+            else valid_oa
         )
         remaining = _clean_polygonal(remaining)
     else:
@@ -60,13 +72,54 @@ def process_oa(
 
     fragments = []
     for pc, parts in result.items():
-        merged = _clean_polygonal(unary_union(parts))
+        merged = _clean_polygonal(safe_union(parts))
         if merged is not None:
             fragments.append((pc, merged))
 
+    # Every postcode with a UPRN seed in this OA must keep at least a minimal
+    # footprint — in a dense OA (a block of flats with hundreds of distinct
+    # postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or
+    # be fully absorbed by a co-located postcode's INSPIRE parcel, producing no
+    # fragment, and an active postcode must never be dropped.
+    orphans = unique_pcs - {pc for pc, _ in fragments}
+    if orphans:
+        fragments.extend(_seed_footprints(orphans, points, postcodes, valid_oa))
+
     return fragments
 
 
+def _seed_footprints(
+    orphans: set[str],
+    points: np.ndarray,
+    postcodes: list[str],
+    valid_oa: Polygon | MultiPolygon,
+) -> list[tuple[str, Polygon | MultiPolygon]]:
+    """Give each orphan postcode a minimal disc footprint at its UPRN seed(s).
+
+    Orphans are postcodes with a UPRN in this OA that nonetheless won no area in
+    the INSPIRE/Voronoi partition. Each keeps a small disc at its true location,
+    clipped to the OA; the overlap with the area's winner is resolved at output.
+    """
+    by_pc: dict[str, list] = defaultdict(list)
+    for i, pc in enumerate(postcodes):
+        if pc in orphans:
+            by_pc[pc].append(points[i])
+
+    out: list[tuple[str, Polygon | MultiPolygon]] = []
+    for pc, pts in by_pc.items():
+        arr = np.asarray(pts, dtype=np.float64)
+        seed = Point(arr[0]) if len(arr) == 1 else MultiPoint(arr)
+        disc = seed.buffer(_MIN_SEED_FOOTPRINT_M)
+        clipped = _clean_polygonal(safe_intersection(disc, valid_oa))
+        if clipped is None:
+            # Seed on/near the OA edge: keep the unclipped disc so the postcode
+            # still gets a footprint at its location rather than no boundary.
+            clipped = _clean_polygonal(disc)
+        if clipped is not None:
+            out.append((pc, clipped))
+    return out
+
+
 def _claim_inspire_parcels(
     valid_oa: Polygon | MultiPolygon,
     points: np.ndarray,
@@ -85,19 +138,42 @@ def _claim_inspire_parcels(
     uprn_pts = shp_points(points)
     pt_idx, cand_idx = cand_tree.query(uprn_pts, predicate="within")
 
-    # First priority: parcels that physically contain UPRNs. Majority vote
-    # resolves blocks of flats or overlapping parcel data.
+    # First priority: parcels that physically contain UPRNs. A parcel holding
+    # UPRNs from a single postcode goes wholly to that postcode. A parcel shared
+    # by several postcodes (a block of flats spanning postcodes, or overlapping
+    # parcel data) is split between them via a sub-Voronoi over their own UPRNs
+    # clipped to the parcel — so EVERY contained postcode keeps part of the
+    # parcel. A bare majority vote would hand the whole parcel to one winner and
+    # leave the losers' UPRNs trapped inside claimed land, dropping them from
+    # both this claim and the `remaining` polygon handed to Voronoi downstream.
     cand_postcodes: dict[int, list[str]] = defaultdict(list)
+    cand_point_idx: dict[int, list[int]] = defaultdict(list)
     for pi, ci in zip(pt_idx, cand_idx):
         cand_postcodes[ci].append(postcodes[pi])
+        cand_point_idx[ci].append(pi)
 
+    points_f64 = points.astype(np.float64, copy=False)
     contained_parts: dict[str, list] = defaultdict(list)
     contained_scores: Counter[str] = Counter()
     for ci, pc_list in cand_postcodes.items():
         pc_counts = Counter(pc_list)
-        winner, votes = pc_counts.most_common(1)[0]
+        if len(pc_counts) == 1:
+            winner = next(iter(pc_counts))
             contained_parts[winner].append(parcels[ci])
-        contained_scores[winner] += votes
+            contained_scores[winner] += pc_counts[winner]
+            continue
+        # Shared parcel: sub-Voronoi over the contained UPRNs so each postcode
+        # present keeps a fragment instead of being absorbed by the winner.
+        sub_idx = cand_point_idx[ci]
+        sub_points = points_f64[sub_idx]
+        sub_postcodes = [postcodes[pi] for pi in sub_idx]
+        for pc, geom in compute_voronoi_regions(
+            sub_points, sub_postcodes, parcels[ci]
+        ).items():
+            cleaned = _clean_polygonal(geom)
+            if cleaned is not None:
+                contained_parts[pc].append(cleaned)
+                contained_scores[pc] += pc_counts[pc]
 
     contained_claimed = _merge_parts_by_postcode(contained_parts)
     contained_claims = sorted(
@@ -109,7 +185,6 @@ def _claim_inspire_parcels(
     # each to the nearest UPRN/postcode so parcel boundaries carry more of the
     # visible postcode shape; Voronoi is then limited to roads, parks, water, and
     # any other non-parcel gaps.
-    points_f64 = points.astype(np.float64, copy=False)
     contained_union = _union_claims(contained_claims)
     nearest_tree = cKDTree(points_f64)
     nearest_parts: dict[str, list] = defaultdict(list)
@@ -119,7 +194,7 @@ def _claim_inspire_parcels(
 
         assignable = parcel
         if contained_union is not None:
-            assignable = assignable.difference(contained_union)
+            assignable = safe_difference(assignable, contained_union)
         for part in _polygon_parts(assignable):
             part = _clean_polygonal(part)
             if part is None:
@@ -147,7 +222,7 @@ def _prepare_inspire_parcels(
             continue
         if not geom.intersects(valid_oa):
             continue
-        clipped = _clean_polygonal(geom.intersection(valid_oa))
+        clipped = _clean_polygonal(safe_intersection(geom, valid_oa))
         if clipped is not None:
             parcels.append(clipped)
     return parcels
@@ -177,7 +252,7 @@ def _merge_parts_by_postcode(
 ) -> dict[str, Polygon | MultiPolygon]:
     merged: dict[str, Polygon | MultiPolygon] = {}
     for pc, parts in parts_by_postcode.items():
-        geom = _clean_polygonal(unary_union(parts))
+        geom = _clean_polygonal(safe_union(parts))
         if geom is not None:
             merged[pc] = geom
     return merged
@@ -188,7 +263,7 @@ def _union_claims(
 ) -> Polygon | MultiPolygon | None:
     if not claims:
         return None
-    return _clean_polygonal(unary_union([geom for _, geom in claims]))
+    return _clean_polygonal(safe_union([geom for _, geom in claims]))
 
 
 def _resolve_ordered_claims(
@@ -202,11 +277,11 @@ def _resolve_ordered_claims(
         if geom is None:
             continue
         if used is not None:
-            geom = _clean_polygonal(geom.difference(used))
+            geom = _clean_polygonal(safe_difference(geom, used))
             if geom is None:
                 continue
         resolved_parts[pc].append(geom)
-        used = _clean_polygonal(geom if used is None else unary_union([used, geom]))
+        used = _clean_polygonal(geom if used is None else safe_union([used, geom]))
     return _merge_parts_by_postcode(resolved_parts)
 
 
@@ -235,11 +310,11 @@ def _extract_polygonal(geom) -> Polygon | MultiPolygon | None:
             return None
         if len(polys) == 1:
             return polys[0]
-        return MultiPolygon(
-            [
-                p
-                for g in polys
-                for p in (g.geoms if g.geom_type == "MultiPolygon" else [g])
-            ]
-        )
+        # Union (not bare MultiPolygon construction): make_valid can emit
+        # overlapping polygonal parts, and a MultiPolygon of overlapping parts is
+        # invalid — it double-counts area and makes the next `.difference()` raise
+        # a TopologyException that aborts the OA (and, in parallel mode, the
+        # worker). safe_union merges them into a valid geometry.
+        merged = safe_union(polys)
+        return merged if not merged.is_empty else None
     return None