andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

lgtm

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-25 22:29:52 +01:00
parent 2efa4d9f47
commit 5e73287eaf
99 changed files with 6392 additions and 1462 deletions
Download patch file Download diff file Expand all files Collapse all files

79
pipeline/transform/test_merge.py
View file

@ -10,14 +10,11 @@ from pipeline.transform.merge import (
LISTED_BUILDING_FEATURE,
TREE_DENSITY_FEATURE,
_LISTING_OVERLAY_SOURCES,
_active_english_postcode_area,
_build_unmatched_listing_seed_rows,
_canonical_postcode_expr,
_best_listing_match,
_coalesce_direct_epc_columns,
_dedupe_collapsed_properties,
_fill_property_level_no_defaults,
_filter_to_active_english_postcodes,
_join_area_side_tables,
_finalize_listings,
_integrate_listings,
@ -31,7 +28,6 @@ from pipeline.transform.merge import (
_matched_listed_building_flags,
_postcode_conservation_area_flags,
_postcode_listed_building_candidates,
_remap_terminated_postcodes,
_split_normal_outputs,
_tree_density_by_postcode,
_validate_lad_source_coverage,
@ -39,6 +35,12 @@ from pipeline.transform.merge import (
_validate_postcode_feature_output,
_validate_property_postcodes,
)
from pipeline.transform.property_base import (
_active_english_postcode_area,
_dedupe_collapsed_properties,
_filter_to_active_english_postcodes,
_remap_terminated_postcodes,
)
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
@ -115,13 +117,16 @@ def test_tree_density_is_area_level_and_survives_the_split() -> None:
assert TREE_DENSITY_FEATURE not in properties_df.columns
def test_crime_columns_are_spatial_counts_not_per_capita() -> None:
# Crime is now a raw spatial count per postcode; the per-1k-residents
# variants were dropped along with the LSOA population denominator.
assert "Serious crime (avg/yr)" in _AREA_COLUMNS
assert "Minor crime (avg/yr)" in _AREA_COLUMNS
assert "Serious crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
assert "Minor crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
def test_crime_columns_are_average_annual_counts() -> None:
# Crime is the average annual recorded incident count (incidents/yr) over
# 7-year and 2-year windows; the old per-1,000 "(per 1k/yr, …)" rate columns
# are gone.
assert "Serious crime (/yr, 7y)" in _AREA_COLUMNS
assert "Serious crime (/yr, 2y)" in _AREA_COLUMNS
assert "Minor crime (/yr, 7y)" in _AREA_COLUMNS
assert "Burglary (/yr, 2y)" in _AREA_COLUMNS
assert "Serious crime (avg/yr)" not in _AREA_COLUMNS
assert "Minor crime (avg/yr)" not in _AREA_COLUMNS
def test_active_english_postcode_area_filters_to_active_england() -> None:
@ -292,8 +297,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
crime = pl.LazyFrame(
{
"postcode": ["AA1 1AA", "BB2 2BB"],
"Serious crime (avg/yr)": [1.0, 2.0],
"Minor crime (avg/yr)": [3.0, 4.0],
"Serious crime (/yr, 7y)": [1.0, 2.0],
"Minor crime (/yr, 7y)": [3.0, 4.0],
}
)
joined = _join_area_side_tables(
@ -343,8 +348,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
crime = pl.LazyFrame(
{
"postcode": ["AB1 2CD", "EF3 4GH"],
"Serious crime (avg/yr)": [1.0, 2.0],
"Minor crime (avg/yr)": [3.0, 4.0],
"Serious crime (/yr, 7y)": [1.0, 2.0],
"Minor crime (/yr, 7y)": [3.0, 4.0],
}
)
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
@ -1314,28 +1319,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
# rollup headlines are precomputed values (deliberately NOT the per-type sum,
# which would be 10.0 each) so this test proves the merge consumes the
# precomputed column rather than re-summing per-type columns.
# rollup rate columns are precomputed in crime_spatial and read straight
# through unchanged (the merge no longer renames or re-sums them).
crime = pl.LazyFrame(
{
"postcode": ["AA1 1AA"],
"Violence and sexual offences (avg/yr)": [1.0],
"Robbery (avg/yr)": [2.0],
"Burglary (avg/yr)": [3.0],
"Possession of weapons (avg/yr)": [4.0],
"Anti-social behaviour (avg/yr)": [1.0],
"Criminal damage and arson (avg/yr)": [1.0],
"Shoplifting (avg/yr)": [1.0],
"Bicycle theft (avg/yr)": [1.0],
"Theft from the person (avg/yr)": [1.0],
"Other theft (avg/yr)": [1.0],
"Vehicle crime (avg/yr)": [1.0],
"Public order (avg/yr)": [1.0],
"Drugs (avg/yr)": [1.0],
"Other crime (avg/yr)": [1.0],
"Serious crime (avg/yr)": [7.5],
"Minor crime (avg/yr)": [4.2],
"Burglary (/yr, 7y)": [3.0],
"Burglary (/yr, 2y)": [3.5],
"Serious crime (/yr, 7y)": [7.5],
"Serious crime (/yr, 2y)": [8.0],
"Minor crime (/yr, 7y)": [4.2],
"Minor crime (/yr, 2y)": [4.6],
}
)
@ -1364,16 +1358,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
by_postcode = {
row["postcode"]: row
for row in joined.select(
"postcode", "serious_crime_avg_yr", "minor_crime_avg_yr"
"postcode",
"Serious crime (/yr, 7y)",
"Minor crime (/yr, 2y)",
).iter_rows(named=True)
}
# Present postcode: rollups are the precomputed headline values, read through
# unchanged (NOT the per-type sum of 10.0).
assert by_postcode["AA1 1AA"]["serious_crime_avg_yr"] == 7.5
assert by_postcode["AA1 1AA"]["minor_crime_avg_yr"] == 4.2
# Missing postcode: rollups stay null rather than fabricating 0.0.
assert by_postcode["BB2 2BB"]["serious_crime_avg_yr"] is None
assert by_postcode["BB2 2BB"]["minor_crime_avg_yr"] is None
# Present postcode: rollup rates pass through unchanged.
assert by_postcode["AA1 1AA"]["Serious crime (/yr, 7y)"] == 7.5
assert by_postcode["AA1 1AA"]["Minor crime (/yr, 2y)"] == 4.6
# Missing postcode: rates stay null rather than fabricating 0.0.
assert by_postcode["BB2 2BB"]["Serious crime (/yr, 7y)"] is None
assert by_postcode["BB2 2BB"]["Minor crime (/yr, 2y)"] is None
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None: