new demo mode & tenure

pipeline/transform/merge.py

@@ -733,14 +733,14 @@ def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None
     postcode resolves into, so a missing LSOA would silently null the ethnicity
     columns for those postcodes; require full coverage instead.
     """
-    iod_lsoas = pl.read_parquet(
-        iod_path, columns=["LSOA code (2021)"]
-    ).rename({"LSOA code (2021)": "lsoa21"})
+    iod_lsoas = pl.read_parquet(iod_path, columns=["LSOA code (2021)"]).rename(
+        {"LSOA code (2021)": "lsoa21"}
+    )
 
     ethnicity_lsoas = pl.read_parquet(ethnicity_path, columns=["lsoa21"])
-    missing_ethnicity = iod_lsoas.join(
-        ethnicity_lsoas, on="lsoa21", how="anti"
-    ).sort("lsoa21")
+    missing_ethnicity = iod_lsoas.join(ethnicity_lsoas, on="lsoa21", how="anti").sort(
+        "lsoa21"
+    )
     if missing_ethnicity.height > 0:
         raise ValueError(
             "Ethnicity data is missing LSOA coverage: "
@@ -749,9 +749,7 @@ def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None
         )
 
 
-def _validate_lad_source_coverage(
-    iod_path: Path, rental_prices_path: Path
-) -> None:
+def _validate_lad_source_coverage(iod_path: Path, rental_prices_path: Path) -> None:
     iod_lads = (
         pl.read_parquet(
             iod_path,
@@ -845,18 +843,32 @@ def _remap_terminated_postcodes(
 
 
 def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
-    """Keep one row per (postcode, pp_address) — the most-recent transaction.
+    """Keep one row per (postcode, address) — the most-recent transaction.
 
     The terminated-postcode remap can map two distinct postcodes onto one active
     successor, collapsing the same physical address onto a single
-    (postcode, pp_address) key with conflicting sale records. Keep the row with
-    the latest date_of_transfer so the headline price/date reflect the most
-    recent transaction; genuinely distinct addresses (a different pp_address) are
-    untouched. pp_address is non-null here (join_epc_pp filters it), so the key
-    never merges unrelated rows.
+    (postcode, address) key with conflicting sale records. Keep the row with the
+    latest date_of_transfer so the headline price/date reflect the most recent
+    transaction; genuinely distinct addresses are untouched.
+
+    The dedup key coalesces the price-paid address with the EPC address: EPC-only
+    dwellings (never sold) have a null pp_address, so keying on pp_address alone
+    would collapse EVERY EPC-only dwelling in a postcode onto one
+    (postcode, null) key and silently drop all but one. Each dwelling's coalesced
+    address is unique within its postcode (the EPC frame is deduped on
+    address+postcode upstream), so the coalesced key keeps them distinct while
+    leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
+    a sale exists.
     """
-    return wide.sort("date_of_transfer", descending=True, nulls_last=True).unique(
-        subset=["postcode", "pp_address"], keep="first", maintain_order=True
+    return (
+        wide.with_columns(
+            pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
+        )
+        .sort("date_of_transfer", descending=True, nulls_last=True)
+        .unique(
+            subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
+        )
+        .drop("_dedupe_address")
     )