..
This commit is contained in:
parent
1ee796b282
commit
ab688243d7
36 changed files with 307 additions and 135 deletions
|
|
@ -83,7 +83,7 @@ _AREA_COLUMNS = [
|
|||
"% Mixed",
|
||||
"% White",
|
||||
"% Other",
|
||||
# Crime — average annual recorded incident count (incidents/yr), 7-year and
|
||||
# Crime: average annual recorded incident count (incidents/yr), 7-year and
|
||||
# 2-year windows. These are the filterable crime features; the per-incident
|
||||
# records live in a separate side table the server loads directly (it bypasses
|
||||
# the merge).
|
||||
|
|
@ -145,11 +145,18 @@ _AREA_COLUMNS = [
|
|||
"% A-levels",
|
||||
"% Degree or higher",
|
||||
"% Other qualifications",
|
||||
# Tenure (Census 2021 TS054, % of households by tenure) — unlike ethnicity &
|
||||
# Tenure (Census 2021 TS054, % of households by tenure). Unlike ethnicity &
|
||||
# education these three percentages are user-filterable, not display-only.
|
||||
"% Owner occupied",
|
||||
"% Social rent",
|
||||
"% Private rent",
|
||||
# Council housing (EPC-derived, NOT Census): share of dwellings in the
|
||||
# postcode ever recorded as social housing per EPC, and the ever-social
|
||||
# subset whose latest EPC certificate is no longer social rented (sold off).
|
||||
# Aggregated from the per-property "was_council_house" / "latest_tenure_status"
|
||||
# flags in _epc_council_by_postcode and joined onto the AREA frame only.
|
||||
"% Council housing",
|
||||
"% Ex-council",
|
||||
# Politics
|
||||
"Voter turnout (%)",
|
||||
"% Labour",
|
||||
|
|
@ -254,7 +261,7 @@ def _subset_numbers_compatible(left: str, right: str) -> bool:
|
|||
Subset (not equality) is correct ONLY for listed-building name matching: a
|
||||
list entry like "10-12 HIGH STREET" should flag "10 HIGH STREET". Address-
|
||||
to-address matching must use the canonical `fuzzy_join._numbers_compatible`
|
||||
instead (set equality over ``\\d+[A-Z]?`` tokens) — subset semantics there
|
||||
instead (set equality over ``\\d+[A-Z]?`` tokens): subset semantics there
|
||||
let a single flat absorb its whole building (see fuzzy_join docstring).
|
||||
"""
|
||||
left_nums = set(_NUMBER_RE.findall(left))
|
||||
|
|
@ -574,7 +581,7 @@ def _is_current_planning_record(end_date: object) -> bool:
|
|||
"""A planning record is current when it has no end-date OR its end-date is
|
||||
still in the future. The planning.data.gov.uk `end-date` field marks when a
|
||||
designation is RETIRED, so a future date (e.g. 2029-12-31) is a still-current
|
||||
area and must NOT be dropped — the previous "any non-empty date = ended"
|
||||
area and must NOT be dropped. The previous "any non-empty date = ended"
|
||||
logic wrongly excluded those (e.g. 22 current Gateshead conservation areas)."""
|
||||
if end_date is None:
|
||||
return True
|
||||
|
|
@ -864,7 +871,7 @@ def _join_area_side_tables(
|
|||
) -> pl.LazyFrame:
|
||||
base = base.join(iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left")
|
||||
# Ethnicity is Census 2021 TS021 at LSOA (~33,755 areas), joined on the same
|
||||
# `lsoa21` key as median age and IoD — a ~100x granularity gain over the old
|
||||
# `lsoa21` key as median age and IoD, a ~100x granularity gain over the old
|
||||
# Local-Authority broadcast, with no change to the 6-bucket output schema.
|
||||
base = base.join(ethnicity, on="lsoa21", how="left")
|
||||
# Education (Census 2021 TS067 "highest level of qualification") is sourced at
|
||||
|
|
@ -1140,9 +1147,9 @@ def _address_score(query: str, candidate: str | None, *, allow_token_set: bool)
|
|||
if not candidate:
|
||||
return 0
|
||||
# token_set_ratio returns 100 whenever the shorter token set is a subset of
|
||||
# the longer. For a NUMBER-LESS query that is unsafe — a single locality
|
||||
# the longer. For a NUMBER-LESS query that is unsafe: a single locality
|
||||
# token (e.g. "KINGSWOOD") subsets to 100 against any long address that
|
||||
# merely contains it — so number-less queries score with token_sort_ratio
|
||||
# merely contains it, so number-less queries score with token_sort_ratio
|
||||
# only, matching the canonical fuzzy_join._score_bucket. For a NUMBERED
|
||||
# query the unconditional fuzzy_join._numbers_compatible gate has already
|
||||
# guaranteed the candidate carries identical house numbers, so token_set
|
||||
|
|
@ -1241,14 +1248,14 @@ def _best_listing_match(
|
|||
``uprn_index`` (postcode-independent, so it is robust even when the
|
||||
listing's postcode is slightly off); (2) failing that, the highest
|
||||
fuzzy street-address similarity within the listing's own postcode bucket.
|
||||
No property-attribute heuristics are used — `fuzzy_join._numbers_compatible`
|
||||
No property-attribute heuristics are used. `fuzzy_join._numbers_compatible`
|
||||
gates every fuzzy match unconditionally (so a number-less listing can never
|
||||
match a numbered property, and vice versa), as in the canonical
|
||||
`fuzzy_join._score_bucket`. A house number additionally lowers the score
|
||||
threshold and (via `_address_score`) permits token_set scoring; a number-less
|
||||
address scores on token_sort only and must match the street almost exactly.
|
||||
The direct-EPC path layers a street-level fallback on top of this strict
|
||||
matcher — see `_best_street_epc_fallback`.
|
||||
matcher. See `_best_street_epc_fallback`.
|
||||
|
||||
``addressed_fields`` names the candidate columns to fuzzy-match against (a
|
||||
candidate may carry both a register and an EPC address). Returns
|
||||
|
|
@ -1301,7 +1308,7 @@ def _best_listing_match(
|
|||
# the listing's own postcode unit is the nearest segment of the street, and a
|
||||
# certificate sharing a house-number token with the listing (e.g. listing
|
||||
# "751 753 Cranbrook Road" vs certificate "751 Cranbrook Road", which fails the
|
||||
# strict set-equality gate) is almost certainly the right property — both
|
||||
# strict set-equality gate) is almost certainly the right property. Both
|
||||
# should beat a bare attribute-agreement win.
|
||||
_STREET_FALLBACK_SAME_POSTCODE_BONUS = 3.0
|
||||
_STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
|
||||
|
|
@ -1309,8 +1316,8 @@ _STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
|
|||
# is era-homogeneous (a single development). When the same-street certificates
|
||||
# span more than this many years between their oldest and newest build, the
|
||||
# street mixes construction eras (a Victorian terrace with modern infill, say),
|
||||
# so no single year represents the unidentified number-less listing property —
|
||||
# the fallback then publishes a null construction year rather than an arbitrary,
|
||||
# so no single year represents the unidentified number-less listing property.
|
||||
# The fallback then publishes a null construction year rather than an arbitrary,
|
||||
# often wrong-by-a-century guess. Two adjacent EPC age bands (one development
|
||||
# straddling a band boundary) span at most ~26 representative years, so this
|
||||
# threshold keeps genuinely-uniform streets while rejecting mixed ones.
|
||||
|
|
@ -1328,7 +1335,7 @@ def _street_match_with_reliable_construction_year(
|
|||
``_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS`` between their oldest and newest
|
||||
build, the street mixes construction eras and the matched certificate's year
|
||||
(e.g. an 1890 Victorian house on a street with 2007 infill) would otherwise
|
||||
be presented as the listing property's own — so the year and its
|
||||
be presented as the listing property's own, so the year and its
|
||||
approximate-date flag are nulled, leaving an honest "unknown" rather than a
|
||||
wrong-by-a-century value. Other street-representative EPC facts (energy
|
||||
rating, floor area) are inherently per-property approximations the fallback
|
||||
|
|
@ -1357,7 +1364,7 @@ def _best_street_epc_fallback(
|
|||
"""Street-level direct-EPC fallback for listings the strict matcher missed.
|
||||
|
||||
~90% of scraped listings publish a street-level address only ("Oldstead
|
||||
Road, Bromley" — Rightmove never exposes the house number or UPRN), so the
|
||||
Road, Bromley", since Rightmove never exposes the house number or UPRN), so the
|
||||
strict matcher in `_best_listing_match` can never match them against the
|
||||
virtually-always-numbered EPC register and their EPC-derived fields
|
||||
(energy rating, interior height, former-council-house flag, construction
|
||||
|
|
@ -1371,7 +1378,7 @@ def _best_street_epc_fallback(
|
|||
same-postcode-unit preference and a house-number-overlap bonus (a
|
||||
numbered listing that failed the strict set-equality gate, e.g. a
|
||||
"751 753" range vs "751", still lands on the right property). The result
|
||||
is street-representative rather than property-exact — hence the distinct
|
||||
is street-representative rather than property-exact, hence the distinct
|
||||
"street" method label so downstream consumers can tell the two confidence
|
||||
levels apart. The matched certificate's construction year is kept only when
|
||||
the street is era-homogeneous (see
|
||||
|
|
@ -1439,7 +1446,7 @@ def _best_street_epc_fallback(
|
|||
# bathrooms (the upstream storage.py defect noted in
|
||||
# `_finalize_listings`). It systematically over-counts, so comparing
|
||||
# it to the EPC habitable-room count biases selection toward larger,
|
||||
# typically older certificates — the opposite of a useful signal —
|
||||
# typically older certificates (the opposite of a useful signal),
|
||||
# and there is no clean listing-side habitable-room count to use.
|
||||
if (
|
||||
listing_postcode
|
||||
|
|
@ -1472,9 +1479,9 @@ def _load_listings_for_merge(listings_path: Path, arcgis_path: Path) -> pl.DataF
|
|||
"""Read the listings parquet and prepare it for the wide-frame merge.
|
||||
|
||||
Output is keyed by `_listing_idx` and carries:
|
||||
* `postcode` — canonical (NSPL `pcds`) form, with terminated postcodes
|
||||
* `postcode`: canonical (NSPL `pcds`) form, with terminated postcodes
|
||||
remapped to their nearest active successor;
|
||||
* `pp_address` — the listing's raw register address (used as the
|
||||
* `pp_address`: the listing's raw register address (used as the
|
||||
address half of the fuzzy match);
|
||||
* one `_actual_*` overlay column per `_LISTING_OVERLAY_SOURCES` entry.
|
||||
"""
|
||||
|
|
@ -1684,7 +1691,7 @@ def _listing_match_frame(listings: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
Listings are matched to EPC certificates and properties by UPRN and by
|
||||
fuzzy street address within their (now accurate, detail-page-sourced)
|
||||
postcode — never by coordinate proximity — so no projected easting/northing
|
||||
postcode (never by coordinate proximity), so no projected easting/northing
|
||||
is computed here. `_listing_uprn` flows through from the loaded listings.
|
||||
"""
|
||||
return listings.with_columns(
|
||||
|
|
@ -1763,8 +1770,8 @@ def _index_candidates(
|
|||
The EPC register's UPRN is NOT unique: a single building/parent UPRN fans
|
||||
across many distinct flats (up to 58 distinct (address, postcode) rows in
|
||||
the 2026-06 data; ~9k UPRNs collide, touching ~20k epc_pp rows). Such a
|
||||
UPRN cannot serve as a 1:1 exact-match key — it would mis-link a listing to
|
||||
one arbitrary flat — so any UPRN that resolves to more than one distinct
|
||||
UPRN cannot serve as a 1:1 exact-match key (it would mis-link a listing to
|
||||
one arbitrary flat), so any UPRN that resolves to more than one distinct
|
||||
``(postcode_key, address_key)`` identity is dropped from ``uprn_index``;
|
||||
those listings fall back to the fuzzy street-address matcher, which
|
||||
disambiguates the specific flat. A UPRN repeated for the SAME identity
|
||||
|
|
@ -1878,7 +1885,7 @@ def _index_epc_streets(
|
|||
maps outcode -> the tokens appearing in at least a quarter of that
|
||||
outcode's street keys. Those are locality suffixes (LONDON, SURREY, the
|
||||
town name) rather than street names, and a fallback match must be anchored
|
||||
by at least one token that is NOT one of them — otherwise a town-only
|
||||
by at least one token that is NOT one of them. Otherwise a town-only
|
||||
listing address ("COULSDON SURREY") token_set-inflates to 100 against any
|
||||
street key carrying the same locality suffix and matches an arbitrary
|
||||
street in the outcode.
|
||||
|
|
@ -2260,7 +2267,7 @@ def _finalize_listings(df: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
@dataclass
|
||||
class _BuildResult:
|
||||
"""Outputs of `_build` — exactly one of the two slot pairs is populated."""
|
||||
"""Outputs of `_build`: exactly one of the two slot pairs is populated."""
|
||||
|
||||
postcode: pl.DataFrame | None = None
|
||||
properties: pl.DataFrame | None = None
|
||||
|
|
@ -2286,6 +2293,38 @@ def _fill_property_level_no_defaults(frame: pl.LazyFrame) -> pl.LazyFrame:
|
|||
)
|
||||
|
||||
|
||||
def _epc_council_by_postcode(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Aggregate the per-property EPC social-tenure flags to POSTCODE percentages.
|
||||
|
||||
Two EPC-derived AREA columns, each a share of *all* deduped dwellings in the
|
||||
postcode (one row per dwelling in ``wide``). This denominator is the dwelling
|
||||
universe, NOT Census households: a dwelling with no EPC is ``was_council_house
|
||||
== "No"`` and so counts against the share, making these EPC-coverage-limited
|
||||
lower bounds that are not directly comparable to the Census ``% Social rent``
|
||||
(which stays at LSOA grain). A postcode holds relatively few dwellings, so
|
||||
these shares are coarse and noisier than an LSOA average.
|
||||
|
||||
* ``% Council housing``: dwellings ever recorded as council/social housing
|
||||
per EPC (``was_council_house == "Yes"``).
|
||||
* ``% Ex-council``: ever-council dwellings whose LATEST EPC certificate is no
|
||||
longer social rented (sold off / no longer social):
|
||||
``was_council_house == "Yes" AND latest_tenure_status != "Rented (social)"``.
|
||||
A null ``latest_tenure_status`` counts as not-currently-social.
|
||||
|
||||
``was_council_house`` is already "Yes"/"No" filled for every row (see
|
||||
``_fill_property_level_no_defaults``), so the means are over the full postcode.
|
||||
Returns a postcode-keyed LazyFrame to left-join onto the AREA frame only.
|
||||
"""
|
||||
currently_social = (pl.col("latest_tenure_status") == "Rented (social)").fill_null(
|
||||
False
|
||||
)
|
||||
ever_social = pl.col("was_council_house") == "Yes"
|
||||
return wide.group_by("postcode").agg(
|
||||
(ever_social.mean() * 100).round(1).alias("% Council housing"),
|
||||
((ever_social & ~currently_social).mean() * 100).round(1).alias("% Ex-council"),
|
||||
)
|
||||
|
||||
|
||||
def _build(
|
||||
epc_pp_path: Path,
|
||||
arcgis_path: Path,
|
||||
|
|
@ -2311,9 +2350,9 @@ def _build(
|
|||
"""Build postcode/properties dataframes (or enriched listings) from epc_pp + auxiliary data.
|
||||
|
||||
Modes:
|
||||
* `normal` — produces (postcode_df, properties_df) as before. Ignores
|
||||
* `normal`: produces (postcode_df, properties_df) as before. Ignores
|
||||
`actual_listings_path` if supplied.
|
||||
* `listings` — requires `actual_listings_path`; produces a single
|
||||
* `listings`: requires `actual_listings_path`; produces a single
|
||||
enriched-listings DataFrame and skips the postcode/properties outputs.
|
||||
Listings flow through the same enrichment joins as historical rows,
|
||||
so postcode-scoped features (tree density, crime, deprivation, …) end
|
||||
|
|
@ -2331,8 +2370,8 @@ def _build(
|
|||
)
|
||||
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
||||
|
||||
# The dwelling universe — floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes — is shared with
|
||||
# The dwelling universe (floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes) is shared with
|
||||
# price estimation so estimates line up 1:1 with these rows. See
|
||||
# pipeline.transform.property_base.
|
||||
wide = build_property_base(epc_pp_path, arcgis_path)
|
||||
|
|
@ -2460,6 +2499,17 @@ def _build(
|
|||
wide = _join_area_side_tables(wide, **area_side_tables)
|
||||
postcode_area = _join_area_side_tables(postcode_area, **area_side_tables)
|
||||
|
||||
# EPC-derived council/ex-council shares: aggregate the per-property social
|
||||
# tenure flags to POSTCODE percentages and attach to the AREA frame only
|
||||
# (these are area columns, like the Census tenure block, not per-property).
|
||||
# Built before dropping latest_tenure_status, which is its only consumer.
|
||||
epc_council_by_postcode = _epc_council_by_postcode(wide)
|
||||
postcode_area = postcode_area.join(epc_council_by_postcode, on="postcode", how="left")
|
||||
# latest_tenure_status is property-grain and not in _AREA_COLUMNS, so the
|
||||
# split would otherwise leak it into properties.parquet. It has served its
|
||||
# purpose (the postcode aggregate above), so drop it from the property frame.
|
||||
wide = wide.drop("latest_tenure_status", strict=False)
|
||||
|
||||
# Derive bedroom count: habitable rooms - 1 (assuming 1 reception room), clipped to 0..4
|
||||
wide = wide.with_columns(
|
||||
(pl.col("number_habitable_rooms") - 1)
|
||||
|
|
@ -2649,7 +2699,7 @@ def main():
|
|||
required=False,
|
||||
help=(
|
||||
"Optional scraped-listings parquet. When provided, listings flow "
|
||||
"through the same merge pipeline as historical properties — set "
|
||||
"through the same merge pipeline as historical properties. Set "
|
||||
"--output-listings to write the enriched-listings file instead "
|
||||
"of the postcode/properties files."
|
||||
),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue