lgtm
This commit is contained in:
parent
2efa4d9f47
commit
5e73287eaf
99 changed files with 6392 additions and 1462 deletions
|
|
@ -29,10 +29,16 @@ from pipeline.utils.fuzzy_join import (
|
|||
normalize_address_key,
|
||||
normalize_postcode_key,
|
||||
)
|
||||
from pipeline.transform.property_base import (
|
||||
MIN_FLOOR_AREA_M2,
|
||||
_active_english_postcode_area,
|
||||
_filter_to_active_english_postcodes,
|
||||
build_property_base,
|
||||
property_type_expr,
|
||||
)
|
||||
from pipeline.utils.normalize import drop_digit_tokens
|
||||
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
||||
|
||||
MIN_FLOOR_AREA_M2 = 10
|
||||
CONSERVATION_AREA_FEATURE = "Within conservation area"
|
||||
# Named "Tree canopy" (not "Street tree") because the underlying density unions
|
||||
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
|
||||
|
|
@ -77,23 +83,42 @@ _AREA_COLUMNS = [
|
|||
"% Mixed",
|
||||
"% White",
|
||||
"% Other",
|
||||
# Crime
|
||||
"Anti-social behaviour (avg/yr)",
|
||||
"Violence and sexual offences (avg/yr)",
|
||||
"Criminal damage and arson (avg/yr)",
|
||||
"Burglary (avg/yr)",
|
||||
"Vehicle crime (avg/yr)",
|
||||
"Robbery (avg/yr)",
|
||||
"Other theft (avg/yr)",
|
||||
"Shoplifting (avg/yr)",
|
||||
"Drugs (avg/yr)",
|
||||
"Possession of weapons (avg/yr)",
|
||||
"Public order (avg/yr)",
|
||||
"Bicycle theft (avg/yr)",
|
||||
"Theft from the person (avg/yr)",
|
||||
"Other crime (avg/yr)",
|
||||
"Serious crime (avg/yr)",
|
||||
"Minor crime (avg/yr)",
|
||||
# Crime — average annual recorded incident count (incidents/yr), 7-year and
|
||||
# 2-year windows. These are the filterable crime features; the per-incident
|
||||
# records live in a separate side table the server loads directly (it bypasses
|
||||
# the merge).
|
||||
"Anti-social behaviour (/yr, 7y)",
|
||||
"Anti-social behaviour (/yr, 2y)",
|
||||
"Violence and sexual offences (/yr, 7y)",
|
||||
"Violence and sexual offences (/yr, 2y)",
|
||||
"Criminal damage and arson (/yr, 7y)",
|
||||
"Criminal damage and arson (/yr, 2y)",
|
||||
"Burglary (/yr, 7y)",
|
||||
"Burglary (/yr, 2y)",
|
||||
"Vehicle crime (/yr, 7y)",
|
||||
"Vehicle crime (/yr, 2y)",
|
||||
"Robbery (/yr, 7y)",
|
||||
"Robbery (/yr, 2y)",
|
||||
"Other theft (/yr, 7y)",
|
||||
"Other theft (/yr, 2y)",
|
||||
"Shoplifting (/yr, 7y)",
|
||||
"Shoplifting (/yr, 2y)",
|
||||
"Drugs (/yr, 7y)",
|
||||
"Drugs (/yr, 2y)",
|
||||
"Possession of weapons (/yr, 7y)",
|
||||
"Possession of weapons (/yr, 2y)",
|
||||
"Public order (/yr, 7y)",
|
||||
"Public order (/yr, 2y)",
|
||||
"Bicycle theft (/yr, 7y)",
|
||||
"Bicycle theft (/yr, 2y)",
|
||||
"Theft from the person (/yr, 7y)",
|
||||
"Theft from the person (/yr, 2y)",
|
||||
"Other crime (/yr, 7y)",
|
||||
"Other crime (/yr, 2y)",
|
||||
"Serious crime (/yr, 7y)",
|
||||
"Serious crime (/yr, 2y)",
|
||||
"Minor crime (/yr, 7y)",
|
||||
"Minor crime (/yr, 2y)",
|
||||
# Amenities
|
||||
"Number of restaurants within 2km",
|
||||
"Number of grocery shops and supermarkets within 2km",
|
||||
|
|
@ -189,8 +214,6 @@ _FINAL_RENAME_COLUMNS = {
|
|||
"outstanding_primary_catchments": "Outstanding primary school catchments",
|
||||
"outstanding_secondary_catchments": "Outstanding secondary school catchments",
|
||||
"max_download_speed": "Max available download speed (Mbps)",
|
||||
"serious_crime_avg_yr": "Serious crime (avg/yr)",
|
||||
"minor_crime_avg_yr": "Minor crime (avg/yr)",
|
||||
"mean_monthly_rent": "Estimated monthly rent",
|
||||
"floor_height": "Interior height (m)",
|
||||
"was_council_house": "Former council house",
|
||||
|
|
@ -822,78 +845,6 @@ def _validate_property_postcodes(df: pl.DataFrame) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Return the supported postcode universe with geography join keys."""
|
||||
return (
|
||||
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
|
||||
.filter(pl.col("doterm").is_null())
|
||||
.select(
|
||||
pl.col("pcds").alias("postcode"),
|
||||
"lat",
|
||||
pl.col("long").alias("lon"),
|
||||
"ctry25cd",
|
||||
pl.col("lsoa21cd").alias("lsoa21"),
|
||||
pl.col("oa21cd").alias("oa21"),
|
||||
pl.col("pcon24cd").alias("pcon"),
|
||||
)
|
||||
.drop_nulls(["postcode"])
|
||||
.unique(["postcode"])
|
||||
)
|
||||
|
||||
|
||||
def _remap_terminated_postcodes(
|
||||
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return (
|
||||
wide.join(
|
||||
postcode_mapping,
|
||||
left_on="postcode",
|
||||
right_on="old_postcode",
|
||||
how="left",
|
||||
)
|
||||
.with_columns(
|
||||
pl.coalesce("new_postcode", "postcode").alias("postcode"),
|
||||
)
|
||||
.drop("new_postcode")
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Keep one row per (postcode, address) — the most-recent transaction.
|
||||
|
||||
The terminated-postcode remap can map two distinct postcodes onto one active
|
||||
successor, collapsing the same physical address onto a single
|
||||
(postcode, address) key with conflicting sale records. Keep the row with the
|
||||
latest date_of_transfer so the headline price/date reflect the most recent
|
||||
transaction; genuinely distinct addresses are untouched.
|
||||
|
||||
The dedup key coalesces the price-paid address with the EPC address: EPC-only
|
||||
dwellings (never sold) have a null pp_address, so keying on pp_address alone
|
||||
would collapse EVERY EPC-only dwelling in a postcode onto one
|
||||
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
|
||||
address is unique within its postcode (the EPC frame is deduped on
|
||||
address+postcode upstream), so the coalesced key keeps them distinct while
|
||||
leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
|
||||
a sale exists.
|
||||
"""
|
||||
return (
|
||||
wide.with_columns(
|
||||
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
|
||||
)
|
||||
.sort("date_of_transfer", descending=True, nulls_last=True)
|
||||
.unique(
|
||||
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
|
||||
)
|
||||
.drop("_dedupe_address")
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_active_english_postcodes(
|
||||
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return wide.join(active_postcodes, on="postcode", how="semi")
|
||||
|
||||
|
||||
def _join_area_side_tables(
|
||||
base: pl.LazyFrame,
|
||||
*,
|
||||
|
|
@ -923,21 +874,15 @@ def _join_area_side_tables(
|
|||
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
|
||||
base = base.join(tenure, on="lsoa21", how="left")
|
||||
|
||||
# Crime is counted spatially per postcode (incidents within 50m of the
|
||||
# postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
|
||||
# precomputes the Serious/Minor headline rollups as the mean of the by-year
|
||||
# rollup bars; read those straight through (renamed to the internal columns
|
||||
# _finalize_merged_columns expects) rather than re-summing the per-type
|
||||
# avg/yr columns — summing divides each type by its OWN years-present and
|
||||
# overstates the rollup when types differ in coverage. A postcode absent from
|
||||
# the crime table keeps null rollups via the left join (no fabricated zero);
|
||||
# the per-type avg/yr columns pass through unchanged for display.
|
||||
base = base.join(crime, on="postcode", how="left").rename(
|
||||
{
|
||||
"Serious crime (avg/yr)": "serious_crime_avg_yr",
|
||||
"Minor crime (avg/yr)": "minor_crime_avg_yr",
|
||||
}
|
||||
)
|
||||
# Crime is counted spatially per postcode (incidents within the boundary
|
||||
# buffer), so it joins on postcode rather than LSOA. crime_spatial writes
|
||||
# average-annual-count columns ("{type} (/yr, 7y|2y)"), including the
|
||||
# Serious/Minor rollups (the exact sum of their components); all pass straight
|
||||
# through to display/filtering. A postcode absent from the crime table keeps
|
||||
# null values via the left join (no fabricated zero). The per-incident records
|
||||
# are a separate side table the server loads directly, so it is not joined
|
||||
# here.
|
||||
base = base.join(crime, on="postcode", how="left")
|
||||
|
||||
base = base.join(median_age, on="lsoa21", how="left")
|
||||
base = base.join(election, on="pcon", how="left")
|
||||
|
|
@ -2386,27 +2331,17 @@ def _build(
|
|||
)
|
||||
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
||||
|
||||
wide = pl.scan_parquet(epc_pp_path).filter(
|
||||
pl.col("total_floor_area").is_null()
|
||||
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
|
||||
)
|
||||
|
||||
# Remap terminated postcodes to nearest active successor before filtering to
|
||||
# the supported active-English postcode universe. Historical properties from
|
||||
# terminated English postcodes are retained under their successor postcode.
|
||||
postcode_mapping = build_postcode_mapping(arcgis_path)
|
||||
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
|
||||
# The remap can collapse two terminated postcodes onto one active successor,
|
||||
# duplicating a physical address's (postcode, pp_address) key; keep only the
|
||||
# most-recent transaction per address before the per-postcode joins.
|
||||
wide = _dedupe_collapsed_properties(wide)
|
||||
# The dwelling universe — floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes — is shared with
|
||||
# price estimation so estimates line up 1:1 with these rows. See
|
||||
# pipeline.transform.property_base.
|
||||
wide = build_property_base(epc_pp_path, arcgis_path)
|
||||
arcgis_raw = pl.scan_parquet(arcgis_path)
|
||||
arcgis = _active_english_postcode_area(arcgis_raw)
|
||||
active_postcodes = arcgis.select("postcode").unique()
|
||||
active_postcode_count = (
|
||||
active_postcodes.select(pl.len()).collect(engine="streaming").item()
|
||||
)
|
||||
wide = _filter_to_active_english_postcodes(wide, active_postcodes)
|
||||
|
||||
if listed_buildings_path is not None:
|
||||
active_postcodes_for_listed = (
|
||||
|
|
@ -2542,37 +2477,10 @@ def _build(
|
|||
how="left",
|
||||
)
|
||||
|
||||
# Derive property_type: prefer EPC data, fall back to price-paid.
|
||||
# For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer detail.
|
||||
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
|
||||
["NO DATA!", "Not Recorded"]
|
||||
)
|
||||
has_epc = pl.col("epc_property_type").is_not_null()
|
||||
is_house = pl.col("epc_property_type") == "House"
|
||||
wide = wide.with_columns(
|
||||
pl.when(has_epc & is_house & ~bad_built_form)
|
||||
.then(pl.col("built_form"))
|
||||
.when(has_epc & is_house)
|
||||
.then(pl.col("pp_property_type"))
|
||||
.when(has_epc)
|
||||
.then(pl.col("epc_property_type"))
|
||||
.otherwise(pl.col("pp_property_type"))
|
||||
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
|
||||
# collapse terrace sub-types, and fold rare types into "Other"
|
||||
.replace(
|
||||
{
|
||||
"Flat": "Flats/Maisonettes",
|
||||
"Maisonette": "Flats/Maisonettes",
|
||||
"End-Terrace": "Terraced",
|
||||
"Mid-Terrace": "Terraced",
|
||||
"Enclosed End-Terrace": "Terraced",
|
||||
"Enclosed Mid-Terrace": "Terraced",
|
||||
"Bungalow": "Other",
|
||||
"Park home": "Other",
|
||||
}
|
||||
)
|
||||
.alias("property_type")
|
||||
)
|
||||
# Derive property_type (EPC preferred, price-paid fallback, built_form for
|
||||
# houses). Shared with price_inputs so the estimate uses the same type; see
|
||||
# property_base.property_type_expr.
|
||||
wide = wide.with_columns(property_type_expr().alias("property_type"))
|
||||
|
||||
wide = wide.with_columns(
|
||||
pl.when(pl.col("duration") == "U")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue