andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Small

Browse source
This commit is contained in:
Andras Schmelczer 2026-07-15 22:00:36 +01:00
parent bce34b73de
commit b47c9ba1ec
13 changed files with 60 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

34
pipeline/transform/merge.py
View file

@ -154,7 +154,8 @@ _AREA_COLUMNS = [
# postcode ever recorded as social housing per EPC, and the ever-social
# subset whose latest EPC certificate is no longer social rented (sold off).
# Aggregated from the per-property "was_council_house" / "latest_tenure_status"
# flags in _epc_council_by_postcode and joined onto the AREA frame only.
# flags in _epc_council_by_postcode, then joined onto the area frame and the
# wide frame (the latter carries them onto listing rows).
"% Council housing",
"% Ex-council",
# Politics
@ -2329,7 +2330,11 @@ def _epc_council_by_postcode(wide: pl.LazyFrame) -> pl.LazyFrame:
``was_council_house`` is already "Yes"/"No" filled for every row (see
``_fill_property_level_no_defaults``), so the means are over the full postcode.
Returns a postcode-keyed LazyFrame to left-join onto the AREA frame only.
Returns a postcode-keyed LazyFrame to left-join onto the area and wide frames.
Call this against the dwelling universe only. `_build` runs it before the
listings splice: unmatched listings append synthetic rows that are not
dwellings, so they must not reach the denominator.
"""
currently_social = (pl.col("latest_tenure_status") == "Rented (social)").fill_null(
False
@ -2420,6 +2425,17 @@ def _build(
pl.lit(None, dtype=pl.Utf8).alias(LISTED_BUILDING_FEATURE)
)
# EPC-derived council/ex-council postcode shares, computed here against the
# pre-splice dwelling universe so both output modes report identical values.
# In listings mode the splice below appends one seed row per unmatched
# listing; those rows would otherwise enter the group_by denominator, and
# since they carry no latest_tenure_status every ever-social one would count
# as ex-council. Reading `wide` here does not rebind it, so the aggregate
# keeps the pre-splice plan; the fill is idempotent with the one below.
epc_council_by_postcode = _epc_council_by_postcode(
_fill_property_level_no_defaults(wide)
)
if actual_listings_path is not None:
wide = _integrate_listings(
wide,
@ -2515,12 +2531,14 @@ def _build(
wide = _join_area_side_tables(wide, **area_side_tables)
postcode_area = _join_area_side_tables(postcode_area, **area_side_tables)
# EPC-derived council/ex-council shares: aggregate the per-property social
# tenure flags to POSTCODE percentages and attach to the AREA frame only
# (these are area columns, like the Census tenure block, not per-property).
# Built before dropping latest_tenure_status, which is its only consumer.
epc_council_by_postcode = _epc_council_by_postcode(wide)
postcode_area = postcode_area.join(epc_council_by_postcode, on="postcode", how="left")
# EPC-derived council/ex-council shares (aggregated above): attach to both
# frames, like every other area side table. The area frame feeds
# postcode.parquet; the wide frame carries them onto listing rows, which are
# projected out of `wide` alone and never see the area frame.
wide = wide.join(epc_council_by_postcode, on="postcode", how="left")
postcode_area = postcode_area.join(
epc_council_by_postcode, on="postcode", how="left"
)
# latest_tenure_status is property-grain and not in _AREA_COLUMNS, so the
# split would otherwise leak it into properties.parquet. It has served its
# purpose (the postcode aggregate above), so drop it from the property frame.