diff --git a/pipeline/transform/merge.py b/pipeline/transform/merge.py index 8e1a0bd..6503fff 100644 --- a/pipeline/transform/merge.py +++ b/pipeline/transform/merge.py @@ -1220,20 +1220,6 @@ def _ratio_bonus( return cap * (1.0 - rel / pct) -def _rooms_bonus(left: int | None, right: int | None) -> float: - if left is None or right is None: - return 0.0 - try: - diff = abs(int(left) - int(right)) - except (TypeError, ValueError): - return 0.0 - if diff == 0: - return 4.0 - if diff == 1: - return 2.0 - return 0.0 - - def _street_only_address(address: str) -> str: """The street/locality part of a normalised address: digit-bearing tokens (house numbers, flat numbers, including letter suffixes like 8A) removed.""" @@ -1350,6 +1336,47 @@ def _best_listing_match( # should beat a bare attribute-agreement win. _STREET_FALLBACK_SAME_POSTCODE_BONUS = 3.0 _STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0 +# A street-representative construction year is only meaningful when the street +# is era-homogeneous (a single development). When the same-street certificates +# span more than this many years between their oldest and newest build, the +# street mixes construction eras (a Victorian terrace with modern infill, say), +# so no single year represents the unidentified number-less listing property — +# the fallback then publishes a null construction year rather than an arbitrary, +# often wrong-by-a-century guess. Two adjacent EPC age bands (one development +# straddling a band boundary) span at most ~26 representative years, so this +# threshold keeps genuinely-uniform streets while rejecting mixed ones. +_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS = 30 + + +def _street_match_with_reliable_construction_year( + match: dict, street_candidates: list[dict] +) -> dict: + """Drop the construction year from a street-fallback match on a mixed street. + + The street fallback identifies a *street*, not the specific (number-less) + listing property, so its construction year is only trustworthy when the + street is era-homogeneous. When the same-street certificates span more than + ``_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS`` between their oldest and newest + build, the street mixes construction eras and the matched certificate's year + (e.g. an 1890 Victorian house on a street with 2007 infill) would otherwise + be presented as the listing property's own — so the year and its + approximate-date flag are nulled, leaving an honest "unknown" rather than a + wrong-by-a-century value. Other street-representative EPC facts (energy + rating, floor area) are inherently per-property approximations the fallback + already accepts and are left untouched. + """ + years = [ + year + for candidate in street_candidates + if (year := candidate.get("_direct_construction_age_band")) is not None + ] + if years and max(years) - min(years) > _STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS: + return { + **match, + "_direct_construction_age_band": None, + "_direct_is_construction_date_approximate": None, + } + return match def _best_street_epc_fallback( @@ -1371,15 +1398,19 @@ def _best_street_epc_fallback( matchable listings (funnel-measured on 2026-06 data). Street identity is token_set_ratio between the digit-stripped halves of both addresses (every same-street certificate scores ~100); qualifying certificates are ranked - by attribute agreement (property type, floor area, habitable rooms) plus - a same-postcode-unit preference and a house-number-overlap bonus (a + by attribute agreement (property type and floor area) plus a + same-postcode-unit preference and a house-number-overlap bonus (a numbered listing that failed the strict set-equality gate, e.g. a "751 753" range vs "751", still lands on the right property). The result is street-representative rather than property-exact — hence the distinct "street" method label so downstream consumers can tell the two confidence - levels apart. Applied to the direct-EPC join only; the property-register - (sale history) join stays strict because a price is property-exact in a - way an energy band is not. + levels apart. The matched certificate's construction year is kept only when + the street is era-homogeneous (see + ``_street_match_with_reliable_construction_year``); on an era-mixed street it + is nulled, since a single year cannot represent the unidentified property. + Applied to the direct-EPC join only; the property-register (sale history) + join stays strict because a price is property-exact in a way an energy band + is not. ``street_score_cache`` memoises the per-(outcode, query-street) fuzzy scan over the outcode's unique street keys: listings on the same street share @@ -1416,6 +1447,7 @@ def _best_street_epc_fallback( listing_postcode = listing.get("_listing_match_postcode") listing_numbers = set(_SUFFIXED_NUMBER_RE.findall(query)) best: dict | None = None + best_street: str | None = None best_total = float("-inf") best_street_score = 0 for street_score, street in qualifying: @@ -1433,10 +1465,13 @@ def _best_street_epc_fallback( pct=0.12, cap=8.0, ) - total += _rooms_bonus( - listing.get("_actual_number_habitable_rooms"), - candidate.get("_direct_number_habitable_rooms"), - ) + # No habitable-room agreement bonus: the listing's only room count is + # "Number of bedrooms & living rooms", which is actually bedrooms + + # bathrooms (the upstream storage.py defect noted in + # `_finalize_listings`). It systematically over-counts, so comparing + # it to the EPC habitable-room count biases selection toward larger, + # typically older certificates — the opposite of a useful signal — + # and there is no clean listing-side habitable-room count to use. if ( listing_postcode and candidate.get("_direct_epc_match_postcode") == listing_postcode @@ -1451,10 +1486,16 @@ def _best_street_epc_fallback( if total > best_total: best_total = total best = candidate + best_street = street best_street_score = street_score if best is None: return None + # A street-representative construction year is unreliable when the winning + # street mixes construction eras; null it rather than imputing one cert's. + best = _street_match_with_reliable_construction_year( + best, outcode_streets.get(best_street or "", []) + ) return best, float(best_street_score), "street", None diff --git a/pipeline/transform/test_merge.py b/pipeline/transform/test_merge.py index f154bb6..dc1f641 100644 --- a/pipeline/transform/test_merge.py +++ b/pipeline/transform/test_merge.py @@ -1129,6 +1129,70 @@ def test_match_direct_epc_street_fallback_recovers_numbered_listing() -> None: assert matches["_direct_epc_match_method"].to_list() == ["street"] +def test_match_direct_epc_street_fallback_nulls_year_on_mixed_era_street() -> None: + # A number-less listing is matched to the *street*, never the exact property. + # When the same-street certificates span construction eras (a Victorian house + # and 2007 infill here), no single year represents the unidentified property, + # so the street-representative construction year is nulled rather than letting + # a new-build inherit an 1890 build date (the Enderby Street, Greenwich bug). + # Other EPC facts stay as street-representative estimates. + matches = _match_direct_epc( + _listing_matches([{"_listing_match_address": "EXAMPLE ROAD BROMLEY"}]), + _direct_epc_candidates( + [ + { + "_direct_epc_match_address": "8 EXAMPLE ROAD", + "_direct_epc_address": "8, Example Road", + "_direct_construction_age_band": 1890, + }, + { + "_direct_epc_row": 1, + "_direct_epc_match_address": "15 EXAMPLE ROAD", + "_direct_epc_address": "15, Example Road", + "_direct_construction_age_band": 2007, + }, + ] + ), + ) + + assert matches.height == 1 + assert matches["_direct_epc_match_method"].to_list() == ["street"] + assert matches["_direct_construction_age_band"].to_list() == [None] + assert matches["_direct_is_construction_date_approximate"].to_list() == [None] + # Energy rating remains a street-representative estimate (only the year is + # categorically wrong on a mixed street; the rest stay best-effort). + assert matches["_direct_current_energy_rating"].to_list() == ["C"] + + +def test_match_direct_epc_street_fallback_keeps_year_on_uniform_street() -> None: + # When the street is era-homogeneous (one development), the + # street-representative construction year IS meaningful and is kept. Two + # adjacent EPC bands a few years apart count as uniform. + matches = _match_direct_epc( + _listing_matches([{"_listing_match_address": "EXAMPLE ROAD BROMLEY"}]), + _direct_epc_candidates( + [ + { + "_direct_epc_match_address": "8 EXAMPLE ROAD", + "_direct_epc_address": "8, Example Road", + "_direct_construction_age_band": 2007, + }, + { + "_direct_epc_row": 1, + "_direct_epc_match_address": "15 EXAMPLE ROAD", + "_direct_epc_address": "15, Example Road", + "_direct_construction_age_band": 2009, + }, + ] + ), + ) + + assert matches.height == 1 + assert matches["_direct_epc_match_method"].to_list() == ["street"] + assert matches["_direct_construction_age_band"].to_list() == [2007] + assert matches["_direct_is_construction_date_approximate"].to_list() == [1] + + def test_match_direct_epc_street_fallback_rejects_town_only_address() -> None: # A town-only listing address ("COULSDON SURREY") shares only the locality # suffix that most street keys in the outcode carry; without a street-name