andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

SPlit up
Some checks failed
CI / Check (push) Failing after 1m58s
Details
Build and publish Docker image / build-and-push (push) Failing after 1m5s
Details

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-12 21:51:37 +01:00
parent cf39ad754e
commit f59d01227b
91 changed files with 10370 additions and 7562 deletions
Download patch file Download diff file Expand all files Collapse all files

25
pipeline/transform/join_epc_pp.py
View file

@ -97,6 +97,13 @@ def epc_band_to_year(band: pl.Expr) -> pl.Expr:
EPC_SOURCE_COLUMNS = [
"address",
# The individual lines behind `address` (= address1+2+3): address2/3
# frequently carry a village/locality token that the price-paid address
# lacks, so the matcher also scores against address1-only and
# address1+address2 variants (see fuzzy_join_on_postcode's variant
# columns).
"address1",
"address2",
"postcode",
"uprn",
"current_energy_rating",
@ -150,6 +157,12 @@ def _select_epc_columns(raw: pl.LazyFrame) -> pl.LazyFrame:
return (
raw.select(
_clean_string("address").alias("epc_address"),
# Match variants: the full address minus the locality-bearing
# trailing lines. Inadmissible variants (ones whose dropped lines
# carry numbers or flat designators) are filtered inside the
# fuzzy join.
_join_address_parts("address1").alias("epc_address_a1"),
_join_address_parts("address1", "address2").alias("epc_address_a12"),
_clean_string("postcode").str.to_uppercase().alias("epc_postcode"),
# UPRN keys an exact listing->EPC join downstream (~99% populated).
_clean_string("uprn").alias("uprn"),
@ -536,6 +549,12 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
.filter(pl.col("pp_property_type") != "Other")
.with_columns(
_join_address_parts("saon", "paon", "street").alias("pp_address"),
# Match variant with the locality appended: the EPC address often
# carries a village/locality token the bare saon+paon+street
# lacks, which alone drags short addresses below the threshold.
_join_address_parts("saon", "paon", "street", "locality").alias(
"pp_address_loc"
),
)
.with_columns(
normalize_address_key(pl.col("pp_address")).alias("_pp_match_address"),
@ -597,6 +616,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
.group_by("_pp_group_address", "_pp_group_postcode", maintain_order=True)
.agg(
pl.col("pp_address").last(),
pl.col("pp_address_loc").last(),
pl.col("postcode").last(),
pl.col("_pp_match_address").last(),
pl.col("_pp_match_postcode").last(),
@ -633,6 +653,8 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
right_address_col="epc_address",
left_postcode_col="postcode",
right_postcode_col="epc_postcode",
left_variant_cols=["pp_address_loc"],
right_variant_cols=["epc_address_a1", "epc_address_a12"],
)
.drop("epc_postcode")
# Audit trail: keep the fuzzy-match confidence (100 = exact address
@ -672,6 +694,9 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
[
"old_new",
"first_transfer_date",
"pp_address_loc",
"epc_address_a1",
"epc_address_a12",
"_pp_match_address",
"_pp_match_postcode",
"_pp_group_address",