lgtm

pipeline/transform/merge.py

@@ -112,6 +112,19 @@ _AREA_COLUMNS = [
     "Outstanding secondary school catchments",
     # Demographics
     "Median age",
+    # Education (Census 2021 TS067, % of residents 16+ by highest qualification)
+    "% No qualifications",
+    "% Some GCSEs",
+    "% Good GCSEs",
+    "% Apprenticeship",
+    "% A-levels",
+    "% Degree or higher",
+    "% Other qualifications",
+    # Tenure (Census 2021 TS054, % of households by tenure) — unlike ethnicity &
+    # education these three percentages are user-filterable, not display-only.
+    "% Owner occupied",
+    "% Social rent",
+    "% Private rent",
     # Politics
     "Voter turnout (%)",
     "% Labour",
@@ -725,28 +738,31 @@ def _tree_density_by_postcode(tree_density_postcodes_path: Path) -> pl.LazyFrame
     )
 
 
-def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None:
-    """Fail if ethnicity (now LSOA-keyed) misses any IoD LSOA.
+def _validate_lsoa_source_coverage(
+    iod_path: Path, lsoa_sources: dict[str, Path]
+) -> None:
+    """Fail if any LSOA-keyed side table misses an IoD LSOA.
 
-    Ethnicity is sourced from Census 2021 TS021 at LSOA, then joined on `lsoa21`
-    like median age and IoD. The IoD table defines the LSOA universe every
-    postcode resolves into, so a missing LSOA would silently null the ethnicity
-    columns for those postcodes; require full coverage instead.
+    Ethnicity (TS021) and education (TS067) are sourced from Census 2021 at LSOA,
+    then joined on `lsoa21` like median age and IoD. The IoD table defines the
+    LSOA universe every postcode resolves into, so a missing LSOA would silently
+    null those columns for the affected postcodes; require full coverage instead.
+    `lsoa_sources` maps a human label (used in the error message) to a parquet
+    that must carry an `lsoa21` column.
     """
     iod_lsoas = pl.read_parquet(iod_path, columns=["LSOA code (2021)"]).rename(
         {"LSOA code (2021)": "lsoa21"}
     )
 
-    ethnicity_lsoas = pl.read_parquet(ethnicity_path, columns=["lsoa21"])
-    missing_ethnicity = iod_lsoas.join(ethnicity_lsoas, on="lsoa21", how="anti").sort(
-        "lsoa21"
-    )
-    if missing_ethnicity.height > 0:
-        raise ValueError(
-            "Ethnicity data is missing LSOA coverage: "
-            f"{missing_ethnicity.height} LSOAs, e.g. "
-            f"{missing_ethnicity.head(10).to_dicts()}"
-        )
+    for label, source_path in lsoa_sources.items():
+        source_lsoas = pl.read_parquet(source_path, columns=["lsoa21"])
+        missing = iod_lsoas.join(source_lsoas, on="lsoa21", how="anti").sort("lsoa21")
+        if missing.height > 0:
+            raise ValueError(
+                f"{label} data is missing LSOA coverage: "
+                f"{missing.height} LSOAs, e.g. "
+                f"{missing.head(10).to_dicts()}"
+            )
 
 
 def _validate_lad_source_coverage(iod_path: Path, rental_prices_path: Path) -> None:
@@ -883,6 +899,8 @@ def _join_area_side_tables(
     *,
     iod: pl.LazyFrame,
     ethnicity: pl.LazyFrame,
+    education: pl.LazyFrame,
+    tenure: pl.LazyFrame,
     crime: pl.LazyFrame,
     median_age: pl.LazyFrame,
     election: pl.LazyFrame,
@@ -898,6 +916,12 @@ def _join_area_side_tables(
     # `lsoa21` key as median age and IoD — a ~100x granularity gain over the old
     # Local-Authority broadcast, with no change to the 6-bucket output schema.
     base = base.join(ethnicity, on="lsoa21", how="left")
+    # Education (Census 2021 TS067 "highest level of qualification") is sourced at
+    # LSOA and joined on the same `lsoa21` key as ethnicity, IoD, and median age.
+    base = base.join(education, on="lsoa21", how="left")
+    # Tenure (Census 2021 TS054 "tenure of household") is sourced at LSOA and
+    # joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
+    base = base.join(tenure, on="lsoa21", how="left")
 
     # Crime is counted spatially per postcode (incidents within 50m of the
     # postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
@@ -2323,6 +2347,8 @@ def _build(
     iod_path: Path,
     poi_proximity_path: Path,
     ethnicity_path: Path,
+    education_path: Path,
+    tenure_path: Path,
     crime_path: Path,
     noise_path: Path,
     school_catchments_path: Path,
@@ -2350,7 +2376,14 @@ def _build(
     """
     if mode == "listings" and actual_listings_path is None:
         raise ValueError("listings mode requires actual_listings_path")
-    _validate_lsoa_source_coverage(iod_path, ethnicity_path)
+    _validate_lsoa_source_coverage(
+        iod_path,
+        {
+            "Ethnicity": ethnicity_path,
+            "Education": education_path,
+            "Tenure": tenure_path,
+        },
+    )
     _validate_lad_source_coverage(iod_path, rental_prices_path)
 
     wide = pl.scan_parquet(epc_pp_path).filter(
@@ -2424,6 +2457,8 @@ def _build(
         *(_less_deprived_percentile_expr(c) for c in _IOD_PERCENTILE_COLUMNS)
     )
     ethnicity = pl.scan_parquet(ethnicity_path)
+    education = pl.scan_parquet(education_path)
+    tenure = pl.scan_parquet(tenure_path)
     crime = pl.scan_parquet(crime_path)
     median_age = pl.scan_parquet(median_age_path)
     election = pl.scan_parquet(election_results_path)
@@ -2475,6 +2510,8 @@ def _build(
     area_side_tables = {
         "iod": iod,
         "ethnicity": ethnicity,
+        "education": education,
+        "tenure": tenure,
         "crime": crime,
         "median_age": median_age,
         "election": election,
@@ -2617,6 +2654,18 @@ def main():
         required=True,
         help="Census 2021 ethnic group (TS021) by LSOA parquet file",
     )
+    parser.add_argument(
+        "--education",
+        type=Path,
+        required=True,
+        help="Census 2021 highest qualification (TS067) by LSOA parquet file",
+    )
+    parser.add_argument(
+        "--tenure",
+        type=Path,
+        required=True,
+        help="Census 2021 household tenure (TS054) by LSOA parquet file",
+    )
     parser.add_argument(
         "--crime",
         type=Path,
@@ -2734,6 +2783,8 @@ def main():
         iod_path=args.iod,
         poi_proximity_path=args.poi_proximity,
         ethnicity_path=args.ethnicity,
+        education_path=args.education,
+        tenure_path=args.tenure,
         crime_path=args.crime,
         noise_path=args.noise,
         school_catchments_path=args.school_catchments,