lgtm

pipeline/transform/property_base.py

@@ -0,0 +1,217 @@
+"""Shared property base: the dwelling universe before any area enrichment.
+
+This is the single source of truth for *which* dwellings exist and their
+intrinsic, source-level attributes (price, floor area, type, addresses). Both
+``merge`` (which enriches it with postcode/LSOA-keyed area features to build
+properties.parquet) and price estimation (which only needs the intrinsic
+columns) start from exactly these rows, so estimates computed here line up 1:1
+with the final properties by the natural key ``(Postcode, coalesced address)``.
+
+Living in its own module is what lets price estimation be *cached* across
+merge changes: ``price_inputs.parquet`` depends only on epc_pp + arcgis + this
+file, so adding an area column to merge.py does not invalidate it and the
+expensive index/kNN steps are skipped.
+"""
+
+import argparse
+from pathlib import Path
+
+import polars as pl
+
+from pipeline.utils.postcode_mapping import build_postcode_mapping
+
+MIN_FLOOR_AREA_M2 = 10
+
+# Columns price estimation reads, with the final (properties.parquet) names so
+# index.py/estimate.py and the join all speak the same schema. The two address
+# columns form the natural join key (Postcode + their coalesce).
+PRICE_INPUT_SELECT = [
+    pl.col("postcode").alias("Postcode"),
+    pl.col("total_floor_area").alias("Total floor area (sqm)"),
+    pl.col("latest_price").alias("Last known price"),
+    pl.col("date_of_transfer").alias("Date of last transaction"),
+    "historical_prices",
+    pl.col("pp_address").alias("Address per Property Register"),
+    pl.col("epc_address").alias("Address per EPC"),
+]
+
+
+def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
+    """Return the supported postcode universe with geography join keys."""
+    return (
+        arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
+        .filter(pl.col("doterm").is_null())
+        .select(
+            pl.col("pcds").alias("postcode"),
+            "lat",
+            pl.col("long").alias("lon"),
+            "ctry25cd",
+            pl.col("lsoa21cd").alias("lsoa21"),
+            pl.col("oa21cd").alias("oa21"),
+            pl.col("pcon24cd").alias("pcon"),
+        )
+        .drop_nulls(["postcode"])
+        .unique(["postcode"])
+    )
+
+
+def _remap_terminated_postcodes(
+    wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
+) -> pl.LazyFrame:
+    return (
+        wide.join(
+            postcode_mapping,
+            left_on="postcode",
+            right_on="old_postcode",
+            how="left",
+        )
+        .with_columns(
+            pl.coalesce("new_postcode", "postcode").alias("postcode"),
+        )
+        .drop("new_postcode")
+    )
+
+
+def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
+    """Keep one row per (postcode, address) — the most-recent transaction.
+
+    The terminated-postcode remap can map two distinct postcodes onto one active
+    successor, collapsing the same physical address onto a single
+    (postcode, address) key with conflicting sale records. Keep the row with the
+    latest date_of_transfer so the headline price/date reflect the most recent
+    transaction; genuinely distinct addresses are untouched.
+
+    The dedup key coalesces the price-paid address with the EPC address: EPC-only
+    dwellings (never sold) have a null pp_address, so keying on pp_address alone
+    would collapse EVERY EPC-only dwelling in a postcode onto one
+    (postcode, null) key and silently drop all but one. Each dwelling's coalesced
+    address is unique within its postcode (the EPC frame is deduped on
+    address+postcode upstream), so the coalesced key keeps them distinct while
+    leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
+    a sale exists.
+    """
+    return (
+        wide.with_columns(
+            pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
+        )
+        .sort("date_of_transfer", descending=True, nulls_last=True)
+        .unique(
+            subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
+        )
+        .drop("_dedupe_address")
+    )
+
+
+def _filter_to_active_english_postcodes(
+    wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
+) -> pl.LazyFrame:
+    return wide.join(active_postcodes, on="postcode", how="semi")
+
+
+def property_type_expr() -> pl.Expr:
+    """Unaliased property-type expression: prefer EPC, fall back to price-paid.
+
+    For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer
+    detail. Depends only on intrinsic base columns (epc_property_type,
+    pp_property_type, built_form), so merge and price_inputs derive the same
+    value. Callers alias it ("property_type" in merge, "Property type" in
+    price_inputs).
+    """
+    bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
+        ["NO DATA!", "Not Recorded"]
+    )
+    has_epc = pl.col("epc_property_type").is_not_null()
+    is_house = pl.col("epc_property_type") == "House"
+    return (
+        pl.when(has_epc & is_house & ~bad_built_form)
+        .then(pl.col("built_form"))
+        .when(has_epc & is_house)
+        .then(pl.col("pp_property_type"))
+        .when(has_epc)
+        .then(pl.col("epc_property_type"))
+        .otherwise(pl.col("pp_property_type"))
+        # Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
+        # collapse terrace sub-types, and fold rare types into "Other"
+        .replace(
+            {
+                "Flat": "Flats/Maisonettes",
+                "Maisonette": "Flats/Maisonettes",
+                "End-Terrace": "Terraced",
+                "Mid-Terrace": "Terraced",
+                "Enclosed End-Terrace": "Terraced",
+                "Enclosed Mid-Terrace": "Terraced",
+                "Bungalow": "Other",
+                "Park home": "Other",
+            }
+        )
+    )
+
+
+def build_postcode_centroids(arcgis_path: Path) -> pl.LazyFrame:
+    """One row per active-English postcode with its lat/lon, from arcgis.
+
+    This is the lat/lon source price estimation needs (index sector centroids,
+    kNN). It is the same per-postcode lat/lon merge writes into postcode.parquet
+    (both come from arcgis), but built straight from arcgis so the index/estimate
+    steps do not depend on the merge output — adding an area column to merge
+    therefore does not invalidate the expensive price index/kNN.
+    """
+    return _active_english_postcode_area(pl.scan_parquet(arcgis_path)).select(
+        pl.col("postcode").alias("Postcode"), "lat", "lon"
+    )
+
+
+def build_property_base(epc_pp_path: Path, arcgis_path: Path) -> pl.LazyFrame:
+    """The deduped, active-English dwelling universe from epc_pp + arcgis.
+
+    Floor filter -> terminated-postcode remap -> collapse-dedupe -> restrict to
+    the active English postcode universe. Returns a LazyFrame with the original
+    epc_pp column names; merge enriches it, the CLI projects it to price_inputs.
+    """
+    wide = pl.scan_parquet(epc_pp_path).filter(
+        pl.col("total_floor_area").is_null()
+        | (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
+    )
+    postcode_mapping = build_postcode_mapping(arcgis_path)
+    wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
+    wide = _dedupe_collapsed_properties(wide)
+    active_postcodes = (
+        _active_english_postcode_area(pl.scan_parquet(arcgis_path))
+        .select("postcode")
+        .unique()
+    )
+    return _filter_to_active_english_postcodes(wide, active_postcodes)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Write the slim price-estimation inputs from epc_pp + arcgis"
+    )
+    parser.add_argument("--epc-pp", type=Path, required=True)
+    parser.add_argument("--arcgis", type=Path, required=True)
+    parser.add_argument(
+        "--output", type=Path, required=True, help="price_inputs.parquet output"
+    )
+    parser.add_argument(
+        "--centroids",
+        type=Path,
+        required=True,
+        help="postcode_centroids.parquet output (Postcode, lat, lon)",
+    )
+    args = parser.parse_args()
+
+    base = build_property_base(args.epc_pp, args.arcgis)
+    price_inputs = base.with_columns(
+        property_type_expr().alias("Property type")
+    ).select(*PRICE_INPUT_SELECT, "Property type")
+    price_inputs.sink_parquet(args.output)
+    n = pl.scan_parquet(args.output).select(pl.len()).collect().item()
+    print(f"Wrote {args.output} ({args.output.stat().st_size / 1e6:.1f} MB), {n:,} dwellings")
+
+    build_postcode_centroids(args.arcgis).sink_parquet(args.centroids)
+    n_pc = pl.scan_parquet(args.centroids).select(pl.len()).collect().item()
+    print(f"Wrote {args.centroids} ({args.centroids.stat().st_size / 1e6:.1f} MB), {n_pc:,} postcodes")
+
+
+if __name__ == "__main__":
+    main()