..

pipeline/transform/join_epc_pp.py

@@ -22,20 +22,20 @@ pl.Config.set_tbl_cols(-1)
 
 RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7}
 # Value-quality floor for price aggregations. A flat nominal floor is a blunt
-# tool against a deflating threshold — £50k was completely normal for a 1990s
+# tool against a deflating threshold: £50k was completely normal for a 1990s
 # house, so a 50k floor wrongly discarded ~a third of legitimate 1990s
 # open-market sales (and deleted properties whose only sales were old/cheap),
 # biasing early-year price history upward. 10k recovers the large [10k,50k)
 # band of genuine cheaper sales while still excluding the nominal/junk transfers
-# (£1 etc.). A small tail of real sub-10k sales is still dropped — a deliberate
+# (£1 etc.). A small tail of real sub-10k sales is still dropped, a deliberate
 # conservative tradeoff to keep clearly-implausible transfers out.
 MIN_PRICE = 10_000
 
 # Time-aware consecutive-sale jump guard. Price-paid contains keyed-in price
-# errors that pass the MIN_PRICE/category filters — e.g. 13 QUICKSETTS HR2 7PP,
-# a 93 m² terrace, sold £140,000 in 2016 then "£207,500,000" in 2026 (clearly
-# £207,500 with extra digits, lodged as category A) — and would otherwise
-# become latest_price. A quality sale is flagged when it exceeds its
+# errors that pass the MIN_PRICE/category filters and would otherwise become
+# latest_price. For example, 13 QUICKSETTS HR2 7PP, a 93 m² terrace, sold
+# £140,000 in 2016 then "£207,500,000" in 2026 (clearly £207,500 with extra
+# digits, lodged as category A). A quality sale is flagged when it exceeds its
 # neighbouring sale by more than JUMP_TOLERANCE * JUMP_GROWTH_PER_YEAR ** years
 # between the two sales. Calibration: genuine extreme appreciation (prime
 # London 1995->2026 is roughly x50 over 31 years) stays comfortably under
@@ -174,8 +174,8 @@ def _clean_number(column: str, dtype: pl.DataType) -> pl.Expr:
 def _join_address_parts(*columns: str) -> pl.Expr:
     """Join address components into one display address, single-spaced.
 
-    Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent —
-    saon is "" on ~88% of rows — and ``concat_str(..., ignore_nulls=True)``
+    Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent
+    (saon is "" on ~88% of rows), and ``concat_str(..., ignore_nulls=True)``
     skips only nulls, so empty components still contributed their separator
     (``' 10 PALACE GREEN'``, doubled spaces when a middle part was empty).
     Convert ``''``→null per component so ignore_nulls works as intended, then
@@ -444,6 +444,13 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
         epc_base.sort("inspection_date", descending=True, nulls_last=True)
         .group_by("_epc_match_address", "_epc_match_postcode")
         .first()
+        # The deduped row carries the most-recent certificate (sorted newest
+        # first, .first() keeps it), so normalising its raw tenure here yields
+        # the LATEST certificate's coarse tenure. Kept past the .drop("tenure")
+        # so downstream (merge) can tell a still-social dwelling from one that
+        # was social once but whose latest cert is no longer social. Computed
+        # before .drop("tenure") because it reads that column.
+        .with_columns(tenure_status(pl.col("tenure")).alias("latest_tenure_status"))
         .drop("tenure")
     )
 
@@ -520,7 +527,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     #
     # Emission rule (walking certificates oldest-first, ignoring unknown-tenure
     # ones so they neither appear nor break the chain):
-    #   - the first known status is emitted only when it is a rental — an
+    #   - the first known status is emitted only when it is a rental: an
     #     owner-occupied baseline is the unremarkable default for a property
     #     that has changed hands and would only clutter the timeline;
     #   - every later certificate is emitted when its status differs from the
@@ -658,8 +665,8 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
         )
         .filter(pl.col("pp_address").is_not_null())
         # Price-paid carries ~72k duplicate (address, postcode, date, price)
-        # transaction groups with DISTINCT transaction ids — the same completed
-        # sale lodged twice — which double-counted sales in historical_prices.
+        # transaction groups with DISTINCT transaction ids (the same completed
+        # sale lodged twice), which double-counted sales in historical_prices.
         # Collapse each to one row. ppd_category stays in the subset so an
         # A/B-categorised pair of the same sale survives as two rows; only the
         # A row feeds the price aggregations (quality_ok), which is intentional.
@@ -714,6 +721,10 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
                 pl.col("date_of_transfer").dt.year().alias("year"),
                 pl.col("date_of_transfer").dt.month().cast(pl.UInt8).alias("month"),
                 "price",
+                # Per-sale new-build flag straight from the PPD "old_new" field
+                # (Y = newly built on first transfer, N = established). Kept on
+                # each transaction so the timeline can mark the new-build sale.
+                (pl.col("old_new") == "Y").alias("is_new"),
             )
             .filter(quality_ok)
             .alias("historical_prices"),