Fable findings in data

pipeline/transform/price_estimation/estimate.py

@@ -25,6 +25,7 @@ from pipeline.transform.price_estimation.knn import (
 )
 from pipeline.transform.price_estimation.utils import (
     CURRENT_FRAC_YEAR,
+    CURRENT_YEAR,
     MAX_LOG_ADJUSTMENT,
     interpolate_log_index,
     sector_expr,
@@ -41,6 +42,87 @@ MIN_KNN_TO_INDEX_RATIO = 0.5
 # only catching outliers.
 MAX_ESTIMATE_TO_LAST_PRICE_RATIO = 20.0
 
+# Guard for rows with NO usable floor area: the per-sqm plausibility check
+# cannot fire there, which let commercial blocks misfiled as dwellings keep
+# absurd headline estimates (e.g. a GBP 175M "Detached" in SW1W). Without
+# floor area we cannot psm-check, so the only sanity reference left is what
+# the local market actually pays: beyond this multiple of the district's
+# recent 99th-percentile sale price the estimate is unreliable and misleading,
+# so it is nulled rather than shown.
+FLOORLESS_ESTIMATE_P99_MULT = 2.0
+# Never null a floorless estimate below this absolute value: genuine mansions
+# in cheap districts can legitimately exceed 2x their district's recent p99,
+# but a sub-GBP 2M estimate is within the plausible single-dwelling range
+# anywhere in the UK, so it survives regardless of the local p99.
+FLOORLESS_ESTIMATE_MIN_CAP = 2_000_000.0
+# Look-back window for the district p99 reference: long enough that thin
+# districts accumulate a usable sale sample, short enough that the reference
+# reflects today's price level rather than a pre-boom one.
+FLOORLESS_P99_LOOKBACK_YEARS = 10
+
+
+def apply_floorless_estimate_guard(df: pl.DataFrame) -> pl.DataFrame:
+    """Null floor-area-less estimates far above their district's recent sales.
+
+    Builds a per-district reference from the SAME frame -- the 99th percentile
+    of `Last known price` over sales in the last FLOORLESS_P99_LOOKBACK_YEARS
+    -- and nulls `Estimated current price` where the floor area is null/zero
+    AND the estimate exceeds max(FLOORLESS_ESTIMATE_P99_MULT * p99,
+    FLOORLESS_ESTIMATE_MIN_CAP). Districts with no recent sales yield a null
+    p99 and are left alone: with neither a psm check nor a local reference we
+    cannot judge the estimate, and nulling on the absolute cap alone would be
+    too aggressive. Expects the `_sector` helper column; rows with floor area
+    present are never touched (the psm guard covers them).
+    """
+    # District = sector minus the trailing sector digit group, matching the
+    # rsplit semantics of utils.hierarchy_keys ("SW1W 9" -> "SW1W").
+    district = pl.col("_sector").str.replace(r"\s+\d+$", "")
+
+    district_p99 = (
+        df.lazy()
+        .filter(
+            pl.col("Last known price").is_not_null(),
+            pl.col("Date of last transaction").dt.year()
+            >= CURRENT_YEAR - FLOORLESS_P99_LOOKBACK_YEARS,
+        )
+        .group_by(district.alias("_district"))
+        .agg(
+            pl.col("Last known price")
+            .cast(pl.Float64)
+            .quantile(0.99)
+            .alias("_district_p99")
+        )
+        .collect()
+    )
+
+    df = df.with_columns(district.alias("_district")).join(
+        district_p99, on="_district", how="left", maintain_order="left"
+    )
+
+    floorless = pl.col("Total floor area (sqm)").is_null() | (
+        pl.col("Total floor area (sqm)") <= 0
+    )
+    cap = pl.max_horizontal(
+        FLOORLESS_ESTIMATE_P99_MULT * pl.col("_district_p99"),
+        pl.lit(FLOORLESS_ESTIMATE_MIN_CAP),
+    )
+    implausible = (
+        pl.col("Estimated current price").is_not_null()
+        & floorless
+        & pl.col("_district_p99").is_not_null()
+        & (pl.col("Estimated current price") > cap)
+    )
+
+    n_nulled = df.select(implausible.sum()).item()
+    print(f"  Floorless-estimate guard: nulled {n_nulled:,} estimates")
+
+    return df.with_columns(
+        pl.when(implausible)
+        .then(None)
+        .otherwise(pl.col("Estimated current price"))
+        .alias("Estimated current price"),
+    ).drop("_district", "_district_p99")
+
 
 def guarded_blend_estimates(
     index_est: np.ndarray,
@@ -249,9 +331,16 @@ def main():
         .alias("Estimated current price"),
     )
 
+    # Floor-area-less rows escape the per-sqm guard above entirely; cap them
+    # against their district's recent sale prices instead (see
+    # apply_floorless_estimate_guard). Must run before temp columns
+    # (_sector) are dropped.
+    df = apply_floorless_estimate_guard(df)
+
     # Derive estimated price per sqm where both estimated price and floor area
     # exist. Now that the implausible-psm estimates are nulled above, the band
-    # filter here mainly guards the floor-area>0 case.
+    # filter here mainly guards the floor-area>0 case. (The floorless guard
+    # never touches floor-area-present rows, so this derivation is unaffected.)
     _est_psm = pl.col("Estimated current price") / pl.col("Total floor area (sqm)")
     df = df.with_columns(
         pl.when(