lgtm

pipeline/transform/crime_hotspot_tiles.py

@@ -53,12 +53,13 @@ def _write_geojsonseq(csvs: list[Path], output_path: Path) -> tuple[int, int]:
     to a shared "map point" anchor, so many incidents land on the exact same
     coordinate. Collapsing them into one feature carrying ``count`` (the number
     of incidents) keeps the per-crime-type and per-month filters intact while
-    turning each hotspot into a single high-weight point. That matters because
-    tippecanoe's ``--drop-densest-as-needed`` thins *feature density*, not
-    weight: with one feature per row the busiest streets were silently deleted;
-    with one weighted feature per anchor those hotspots survive and the dropped
-    detail is only redundant duplicate points. The heatmap reads ``count`` as
-    its weight.
+    turning each hotspot into a single high-weight point. That matters for the
+    heatmap weight: each anchor becomes one high-weight point, and tippecanoe's
+    ``--cluster-densest-as-needed`` (with ``--accumulate-attribute=count:sum``)
+    merges any still-too-dense low-zoom features by *summing* their counts
+    rather than dropping them, so the total heat weight is conserved across zoom
+    levels and the surface no longer jumps at tile-zoom boundaries. The heatmap
+    reads ``count`` as its weight.
     """
     grouped = (
         pl.scan_csv(
@@ -144,7 +145,18 @@ def build_crime_hotspot_tiles(
                 str(min_zoom),
                 "--maximum-zoom",
                 str(max_zoom),
-                "--drop-densest-as-needed",
+                # Merge (don't delete) the densest features at low zoom and sum
+                # their incident counts into the surviving point, so total heat
+                # weight is conserved across zoom levels. With --drop-densest the
+                # z14 tile lost hotspots that z15 kept, so the heatmap visibly
+                # collapsed into smaller spots when crossing the z14<->z15 tile
+                # boundary. Clustering is spatial only (it can merge different
+                # crime_types into one representative point), so per-type
+                # filtering is slightly approximate in the densest z14 tiles;
+                # the all-types surface and every zoom >= 15 stay accurate.
+                "--cluster-densest-as-needed",
+                "--accumulate-attribute=count:sum",
+                "--accumulate-attribute=weight:sum",
                 "--extend-zooms-if-still-dropping",
                 "--temporary-directory",
                 tmp,

pipeline/transform/merge.py

@@ -112,6 +112,19 @@ _AREA_COLUMNS = [
     "Outstanding secondary school catchments",
     # Demographics
     "Median age",
+    # Education (Census 2021 TS067, % of residents 16+ by highest qualification)
+    "% No qualifications",
+    "% Some GCSEs",
+    "% Good GCSEs",
+    "% Apprenticeship",
+    "% A-levels",
+    "% Degree or higher",
+    "% Other qualifications",
+    # Tenure (Census 2021 TS054, % of households by tenure) — unlike ethnicity &
+    # education these three percentages are user-filterable, not display-only.
+    "% Owner occupied",
+    "% Social rent",
+    "% Private rent",
     # Politics
     "Voter turnout (%)",
     "% Labour",
@@ -725,28 +738,31 @@ def _tree_density_by_postcode(tree_density_postcodes_path: Path) -> pl.LazyFrame
     )
 
 
-def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None:
-    """Fail if ethnicity (now LSOA-keyed) misses any IoD LSOA.
+def _validate_lsoa_source_coverage(
+    iod_path: Path, lsoa_sources: dict[str, Path]
+) -> None:
+    """Fail if any LSOA-keyed side table misses an IoD LSOA.
 
-    Ethnicity is sourced from Census 2021 TS021 at LSOA, then joined on `lsoa21`
-    like median age and IoD. The IoD table defines the LSOA universe every
-    postcode resolves into, so a missing LSOA would silently null the ethnicity
-    columns for those postcodes; require full coverage instead.
+    Ethnicity (TS021) and education (TS067) are sourced from Census 2021 at LSOA,
+    then joined on `lsoa21` like median age and IoD. The IoD table defines the
+    LSOA universe every postcode resolves into, so a missing LSOA would silently
+    null those columns for the affected postcodes; require full coverage instead.
+    `lsoa_sources` maps a human label (used in the error message) to a parquet
+    that must carry an `lsoa21` column.
     """
     iod_lsoas = pl.read_parquet(iod_path, columns=["LSOA code (2021)"]).rename(
         {"LSOA code (2021)": "lsoa21"}
     )
 
-    ethnicity_lsoas = pl.read_parquet(ethnicity_path, columns=["lsoa21"])
-    missing_ethnicity = iod_lsoas.join(ethnicity_lsoas, on="lsoa21", how="anti").sort(
-        "lsoa21"
-    )
-    if missing_ethnicity.height > 0:
-        raise ValueError(
-            "Ethnicity data is missing LSOA coverage: "
-            f"{missing_ethnicity.height} LSOAs, e.g. "
-            f"{missing_ethnicity.head(10).to_dicts()}"
-        )
+    for label, source_path in lsoa_sources.items():
+        source_lsoas = pl.read_parquet(source_path, columns=["lsoa21"])
+        missing = iod_lsoas.join(source_lsoas, on="lsoa21", how="anti").sort("lsoa21")
+        if missing.height > 0:
+            raise ValueError(
+                f"{label} data is missing LSOA coverage: "
+                f"{missing.height} LSOAs, e.g. "
+                f"{missing.head(10).to_dicts()}"
+            )
 
 
 def _validate_lad_source_coverage(iod_path: Path, rental_prices_path: Path) -> None:
@@ -883,6 +899,8 @@ def _join_area_side_tables(
     *,
     iod: pl.LazyFrame,
     ethnicity: pl.LazyFrame,
+    education: pl.LazyFrame,
+    tenure: pl.LazyFrame,
     crime: pl.LazyFrame,
     median_age: pl.LazyFrame,
     election: pl.LazyFrame,
@@ -898,6 +916,12 @@ def _join_area_side_tables(
     # `lsoa21` key as median age and IoD — a ~100x granularity gain over the old
     # Local-Authority broadcast, with no change to the 6-bucket output schema.
     base = base.join(ethnicity, on="lsoa21", how="left")
+    # Education (Census 2021 TS067 "highest level of qualification") is sourced at
+    # LSOA and joined on the same `lsoa21` key as ethnicity, IoD, and median age.
+    base = base.join(education, on="lsoa21", how="left")
+    # Tenure (Census 2021 TS054 "tenure of household") is sourced at LSOA and
+    # joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
+    base = base.join(tenure, on="lsoa21", how="left")
 
     # Crime is counted spatially per postcode (incidents within 50m of the
     # postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
@@ -2323,6 +2347,8 @@ def _build(
     iod_path: Path,
     poi_proximity_path: Path,
     ethnicity_path: Path,
+    education_path: Path,
+    tenure_path: Path,
     crime_path: Path,
     noise_path: Path,
     school_catchments_path: Path,
@@ -2350,7 +2376,14 @@ def _build(
     """
     if mode == "listings" and actual_listings_path is None:
         raise ValueError("listings mode requires actual_listings_path")
-    _validate_lsoa_source_coverage(iod_path, ethnicity_path)
+    _validate_lsoa_source_coverage(
+        iod_path,
+        {
+            "Ethnicity": ethnicity_path,
+            "Education": education_path,
+            "Tenure": tenure_path,
+        },
+    )
     _validate_lad_source_coverage(iod_path, rental_prices_path)
 
     wide = pl.scan_parquet(epc_pp_path).filter(
@@ -2424,6 +2457,8 @@ def _build(
         *(_less_deprived_percentile_expr(c) for c in _IOD_PERCENTILE_COLUMNS)
     )
     ethnicity = pl.scan_parquet(ethnicity_path)
+    education = pl.scan_parquet(education_path)
+    tenure = pl.scan_parquet(tenure_path)
     crime = pl.scan_parquet(crime_path)
     median_age = pl.scan_parquet(median_age_path)
     election = pl.scan_parquet(election_results_path)
@@ -2475,6 +2510,8 @@ def _build(
     area_side_tables = {
         "iod": iod,
         "ethnicity": ethnicity,
+        "education": education,
+        "tenure": tenure,
         "crime": crime,
         "median_age": median_age,
         "election": election,
@@ -2617,6 +2654,18 @@ def main():
         required=True,
         help="Census 2021 ethnic group (TS021) by LSOA parquet file",
     )
+    parser.add_argument(
+        "--education",
+        type=Path,
+        required=True,
+        help="Census 2021 highest qualification (TS067) by LSOA parquet file",
+    )
+    parser.add_argument(
+        "--tenure",
+        type=Path,
+        required=True,
+        help="Census 2021 household tenure (TS054) by LSOA parquet file",
+    )
     parser.add_argument(
         "--crime",
         type=Path,
@@ -2734,6 +2783,8 @@ def main():
         iod_path=args.iod,
         poi_proximity_path=args.poi_proximity,
         ethnicity_path=args.ethnicity,
+        education_path=args.education,
+        tenure_path=args.tenure,
         crime_path=args.crime,
         noise_path=args.noise,
         school_catchments_path=args.school_catchments,

pipeline/transform/test_merge.py

@@ -300,6 +300,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
         base,
         iod=pl.LazyFrame({"LSOA code (2021)": ["E01000001", "E01000002"]}),
         ethnicity=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        education=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        tenure=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         crime=crime,
         median_age=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         election=pl.LazyFrame({"pcon": ["E14000001", "E14000002"]}),
@@ -358,6 +360,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
         base,
         iod=pl.LazyFrame({"LSOA code (2021)": ["E01000001", "E01000002"]}),
         ethnicity=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        education=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        tenure=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         crime=crime,
         median_age=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         election=pl.LazyFrame({"pcon": ["E14000001", "E14000002"]}),
@@ -586,7 +590,7 @@ def test_validate_lsoa_source_coverage_allows_full_ethnicity_coverage(
         ethnicity_path
     )
 
-    _validate_lsoa_source_coverage(iod_path, ethnicity_path)
+    _validate_lsoa_source_coverage(iod_path, {"Ethnicity": ethnicity_path})
 
 
 def test_validate_lsoa_source_coverage_rejects_missing_lsoa(tmp_path) -> None:
@@ -598,7 +602,7 @@ def test_validate_lsoa_source_coverage_rejects_missing_lsoa(tmp_path) -> None:
     pl.DataFrame({"lsoa21": ["E01000001"]}).write_parquet(ethnicity_path)
 
     with pytest.raises(ValueError, match="Ethnicity data is missing LSOA coverage"):
-        _validate_lsoa_source_coverage(iod_path, ethnicity_path)
+        _validate_lsoa_source_coverage(iod_path, {"Ethnicity": ethnicity_path})
 
 
 def test_tree_density_by_postcode_aliases_radius_percentile(tmp_path) -> None:
@@ -1339,6 +1343,8 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
         base,
         iod=pl.LazyFrame({"LSOA code (2021)": ["E01000001", "E01000002"]}),
         ethnicity=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        education=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
+        tenure=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         crime=crime,
         median_age=pl.LazyFrame({"lsoa21": ["E01000001", "E01000002"]}),
         election=pl.LazyFrame({"pcon": ["E14000001", "E14000002"]}),