lgtm
This commit is contained in:
parent
2efa4d9f47
commit
5e73287eaf
99 changed files with 6392 additions and 1462 deletions
212
pipeline/transform/area_crime_averages.py
Normal file
212
pipeline/transform/area_crime_averages.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
"""Precompute per-outcode / per-postcode-sector / national mean headline crime
|
||||
counts for the right pane's area comparison.
|
||||
|
||||
The right pane shows each crime metric next to its area context: the mean
|
||||
average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
|
||||
``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
|
||||
within a postcode (the merge keys it on the postcode), so each postcode
|
||||
contributes its single value weighted by how many properties sit in it — keeping
|
||||
every scope on the same property-weighted basis as the per-selection mean, so the
|
||||
four numbers (this selection / sector / outcode / nation) are directly
|
||||
comparable. The national figure here is an EXACT property-weighted mean, which is
|
||||
why it overrides the upward-biased histogram-bin national average for crime.
|
||||
|
||||
This used to be recomputed inside the server on every boot from the loaded
|
||||
property matrix. It is a pure function of the two merge outputs, so it belongs in
|
||||
the data build; the server now just loads the parquet this writes. Reading the
|
||||
crime values from ``postcode.parquet`` and the per-postcode property weights from
|
||||
``properties.parquet`` mirrors exactly the two inputs the server loads, so the
|
||||
result matches what the server used to compute (minus its u16 quantization loss).
|
||||
|
||||
Output schema — one row per area:
|
||||
|
||||
scope : ``"national"`` | ``"outcode"`` | ``"sector"``
|
||||
area : the outcode (``"E14"``) / sector (``"E14 2"``);
|
||||
``""`` for the single national row
|
||||
``<type> (/yr, 7y|2y)`` : Float32 property-weighted mean crime count per year
|
||||
(null = the scope has no data for that type)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
# Filterable crime columns are the average-annual incident counts and carry this
|
||||
# marker in postcode.parquet (e.g. "Burglary (/yr, 7y)"). We average those. The
|
||||
# full column name is kept; the server discovers and keys area averages by the
|
||||
# same names.
|
||||
COUNT_MARKER = " (/yr, "
|
||||
|
||||
# `scope` discriminator values. The server's loader dispatches on these.
|
||||
SCOPE_NATIONAL = "national"
|
||||
SCOPE_OUTCODE = "outcode"
|
||||
SCOPE_SECTOR = "sector"
|
||||
|
||||
# Area label on the national row — it spans the whole country, so it has no code.
|
||||
NATIONAL_AREA = ""
|
||||
|
||||
# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
|
||||
# "E14 2DG").
|
||||
POSTCODE_COLUMN = "Postcode"
|
||||
|
||||
# Internal weight / split columns dropped before write.
|
||||
_WEIGHT_COLUMN = "_weight"
|
||||
_OUTCODE_COLUMN = "_outcode"
|
||||
_SECTOR_COLUMN = "_sector"
|
||||
|
||||
|
||||
def _crime_columns(columns: list[str]) -> list[str]:
|
||||
crime_cols = [name for name in columns if COUNT_MARKER in name]
|
||||
if not crime_cols:
|
||||
raise ValueError(
|
||||
f"postcode parquet has no '*{COUNT_MARKER}*' crime count columns to average"
|
||||
)
|
||||
return crime_cols
|
||||
|
||||
|
||||
def _weighted_mean(column: str) -> pl.Expr:
|
||||
"""Property-weighted mean of ``column`` that excludes nulls from BOTH the
|
||||
value sum and the weight.
|
||||
|
||||
A null crime value is a genuine gap (the postcode's police force published no
|
||||
usable data), not zero crime, so it must dilute neither the numerator nor the
|
||||
denominator — exactly as the server's former estimator skipped NaN values.
|
||||
Yields null when no postcode in the group has data for this type.
|
||||
"""
|
||||
weight = pl.col(_WEIGHT_COLUMN)
|
||||
numerator = (pl.col(column) * weight).sum()
|
||||
denominator = weight.filter(pl.col(column).is_not_null()).sum()
|
||||
return (
|
||||
pl.when(denominator > 0)
|
||||
.then(numerator / denominator)
|
||||
.otherwise(None)
|
||||
.cast(pl.Float32)
|
||||
.alias(column)
|
||||
)
|
||||
|
||||
|
||||
def compute_area_crime_averages(
|
||||
postcodes: pl.LazyFrame, properties: pl.LazyFrame
|
||||
) -> pl.DataFrame:
|
||||
"""Build the national / per-outcode / per-sector crime-average table.
|
||||
|
||||
``postcodes`` is the merge's postcode output (one row per active postcode,
|
||||
carrying the ``"* (/yr, *)"`` crime count columns); ``properties`` is the merge's
|
||||
per-property output, used only to weight each postcode by its property count.
|
||||
"""
|
||||
crime_cols = _crime_columns(postcodes.collect_schema().names())
|
||||
|
||||
# Property weight per postcode = how many property rows the server indexes
|
||||
# under it. The inner join keeps only postcodes that actually carry
|
||||
# properties, matching the server's per-postcode row index (a postcode with
|
||||
# no properties never contributed to any average).
|
||||
weights = properties.group_by(POSTCODE_COLUMN).agg(pl.len().alias(_WEIGHT_COLUMN))
|
||||
|
||||
# Outcode / sector of the spaced `pcds` postcode, matching the server's
|
||||
# postcode_outcode / postcode_sector (split on the single space; sector =
|
||||
# outcode + space + first inward character). Null where the form has no
|
||||
# inward code, so such rows drop out of the per-area groups.
|
||||
parts = pl.col(POSTCODE_COLUMN).str.splitn(" ", 2).struct
|
||||
outward = parts.field("field_0")
|
||||
inward = parts.field("field_1")
|
||||
base = (
|
||||
postcodes.select(POSTCODE_COLUMN, *crime_cols)
|
||||
.join(weights, on=POSTCODE_COLUMN, how="inner")
|
||||
.with_columns(
|
||||
pl.when(inward.is_not_null())
|
||||
.then(outward)
|
||||
.otherwise(None)
|
||||
.alias(_OUTCODE_COLUMN),
|
||||
pl.when(inward.str.len_chars() >= 1)
|
||||
.then(outward + pl.lit(" ") + inward.str.slice(0, 1))
|
||||
.otherwise(None)
|
||||
.alias(_SECTOR_COLUMN),
|
||||
)
|
||||
)
|
||||
|
||||
mean_exprs = [_weighted_mean(column) for column in crime_cols]
|
||||
|
||||
national = base.select(
|
||||
pl.lit(SCOPE_NATIONAL).alias("scope"),
|
||||
pl.lit(NATIONAL_AREA).alias("area"),
|
||||
*mean_exprs,
|
||||
)
|
||||
by_outcode = (
|
||||
base.drop_nulls(_OUTCODE_COLUMN)
|
||||
.group_by(_OUTCODE_COLUMN)
|
||||
.agg(mean_exprs)
|
||||
.select(
|
||||
pl.lit(SCOPE_OUTCODE).alias("scope"),
|
||||
pl.col(_OUTCODE_COLUMN).alias("area"),
|
||||
*crime_cols,
|
||||
)
|
||||
)
|
||||
by_sector = (
|
||||
base.drop_nulls(_SECTOR_COLUMN)
|
||||
.group_by(_SECTOR_COLUMN)
|
||||
.agg(mean_exprs)
|
||||
.select(
|
||||
pl.lit(SCOPE_SECTOR).alias("scope"),
|
||||
pl.col(_SECTOR_COLUMN).alias("area"),
|
||||
*crime_cols,
|
||||
)
|
||||
)
|
||||
|
||||
result = pl.concat([national, by_outcode, by_sector], how="vertical").collect(
|
||||
engine="streaming"
|
||||
)
|
||||
|
||||
# Drop per-area rows where every crime type is null: the server only created
|
||||
# a map entry once a scope had at least one finite value, so an all-null
|
||||
# outcode/sector reported no code at all. The national row is always kept (it
|
||||
# always has data, and is emitted even for areas absent from both maps).
|
||||
has_any = pl.any_horizontal(pl.col(column).is_not_null() for column in crime_cols)
|
||||
return result.filter((pl.col("scope") == SCOPE_NATIONAL) | has_any)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Precompute national / per-outcode / per-sector mean headline crime "
|
||||
"counts from the merge outputs"
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--postcodes",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="postcode.parquet (area features, incl. the '* (/yr, *)' crime columns)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--properties",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="properties.parquet (per-property rows; supplies postcode property weights)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output area_crime_averages.parquet path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = compute_area_crime_averages(
|
||||
pl.scan_parquet(args.postcodes), pl.scan_parquet(args.properties)
|
||||
)
|
||||
outcodes = result.filter(pl.col("scope") == SCOPE_OUTCODE).height
|
||||
sectors = result.filter(pl.col("scope") == SCOPE_SECTOR).height
|
||||
print(
|
||||
f"Area crime averages: {result.height} rows "
|
||||
f"({outcodes} outcodes, {sectors} sectors, "
|
||||
f"{len(_crime_columns(result.columns))} crime types)"
|
||||
)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
result.write_parquet(args.output, compression="zstd")
|
||||
print(f"Saved to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -2,22 +2,26 @@
|
|||
|
||||
Instead of attributing each incident to its published LSOA code, this transform
|
||||
counts the anonymised incident *points* that fall within ``buffer_m`` (default
|
||||
100m) of each postcode's boundary polygon (the polygon buffered outward). A point
|
||||
50m) of each postcode's boundary polygon (the polygon buffered outward). A point
|
||||
inside several overlapping buffers counts for each postcode -- the same
|
||||
multiplicity the tree-density filter uses for features near more than one
|
||||
postcode. The wide 100m buffer deliberately smooths police.uk's snap-to-grid
|
||||
postcode. The 50m buffer deliberately smooths police.uk's snap-to-grid
|
||||
coordinates, which would otherwise make the count hypersensitive to which side of
|
||||
a narrow line a shared "map point" anchor happened to land on.
|
||||
|
||||
Counts are **area-normalised**: each postcode's count is divided by its buffered
|
||||
catchment area and rescaled by the median catchment area, so the metric reflects
|
||||
crime *density* rather than how much ground the buffer sweeps (a median-sized
|
||||
catchment is left unchanged; a large rural postcode is no longer inflated simply
|
||||
for covering more of the map). Normalising by the buffered area -- the region
|
||||
that actually collects points -- rather than the raw polygon keeps tiny unit
|
||||
postcodes from being over-inflated by the fixed buffer-ring floor. NOTE: this is
|
||||
an incident *density of the surrounding streets*, not a per-resident risk --
|
||||
zero-resident commercial centres (Soho, retail parks) legitimately rank high.
|
||||
One figure is produced for every postcode and crime type, averaged over the
|
||||
**last 7 years** and the **last 2 years**:
|
||||
|
||||
* the **average annual incident count** -- ``sum(counts in covered window) * 12 /
|
||||
covered_months`` -- the raw, absolute number of recorded incidents per year,
|
||||
with no per-area or per-capita normalisation. A covered postcode with no
|
||||
incidents of a type gets ``0``; a postcode whose force never published in the
|
||||
window, or whose geometry is unusable, gets a *null* (unknown, never a zero).
|
||||
|
||||
This figure is what the server exposes as the filterable crime feature -- the
|
||||
headline metric in the right pane. (An earlier version divided it by an ambient
|
||||
daytime population to get a per-1,000-people rate; that was hard to read, so the
|
||||
absolute per-year count is now used directly.)
|
||||
|
||||
**Force-coverage calendar.** police.uk has multi-year publication gaps for whole
|
||||
forces (Greater Manchester has published nothing between 2019-07 and the present
|
||||
|
|
@ -29,28 +33,25 @@ computed against the months the postcode's own force actually published:
|
|||
matched it (BTP, which reports nationwide, is excluded from the vote);
|
||||
postcodes with no incidents inherit their outcode's majority force, then the
|
||||
national modal force.
|
||||
* The headline ``"{type} (avg/yr)"`` is the POOLED annualised rate over the
|
||||
force's covered months: ``sum(counts in covered years) * 12 / covered_months``.
|
||||
Years in which the force published nothing contribute neither incidents nor
|
||||
months, so a coverage gap no longer reads as a low-crime period. (Pooling over
|
||||
covered months also fixes the old "divide by years-with-incidents" headline,
|
||||
which inflated sporadic categories by up to ~15x.)
|
||||
* The by-year series only emits bars for years with at least
|
||||
``min_bar_months`` covered months (default 6): annualising a single observed
|
||||
month x12 produced misleading spikes. Each bar is scaled by the force's
|
||||
covered months in that year, not the global month calendar.
|
||||
* A window's average pools the counts over the force's *covered* months inside
|
||||
that window and annualises by those months, so a coverage gap shrinks the data
|
||||
rather than reading as a low-crime dip.
|
||||
* The by-year series only emits bars for years with at least ``min_bar_months``
|
||||
covered months (default 6).
|
||||
* ``covered_years`` (list[struct{year, months}]) is written for every postcode
|
||||
so the server can tell "covered, zero crime" (year listed, no bar) from "no
|
||||
data" (year absent) instead of charting gaps as zeros.
|
||||
so the server can tell "covered, zero crime" from "no data".
|
||||
* Postcodes whose boundary buffer is unusable (broken geometry) get null
|
||||
headline columns and an empty ``covered_years`` -- unknown, not zero.
|
||||
figures and an empty ``covered_years`` -- unknown, not zero.
|
||||
|
||||
Outputs mirror the old LSOA transform's shape but are keyed on ``postcode``:
|
||||
Outputs, all keyed on ``postcode``:
|
||||
|
||||
* ``crime_by_postcode.parquet`` -- ``postcode`` + ``"{type} (avg/yr)"`` columns.
|
||||
* ``crime_by_postcode_by_year.parquet`` -- one row per postcode: ``postcode`` +
|
||||
``covered_years`` + nested ``"{type} (by year)"`` ``list[struct{year, count}]``
|
||||
columns, with Serious/Minor rollups.
|
||||
* ``crime_by_postcode.parquet`` -- ``"{type} (/yr, 7y)"`` / ``"{type} (/yr, 2y)"``
|
||||
average-annual-count columns (the filterable crime features).
|
||||
* ``crime_by_postcode_by_year.parquet`` -- ``covered_years`` + nested
|
||||
``"{type} (by year)"`` ``list[struct{year, count}]`` per-year raw counts.
|
||||
* ``crime_records.parquet`` -- one row per counted incident over the last 7
|
||||
years (``postcode`` + month/type/location/outcome/lat/lon), sorted by
|
||||
postcode so the server can slice each postcode's incidents directly.
|
||||
|
||||
Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
|
||||
points", not true locations, and a share of rows have no coordinate at all
|
||||
|
|
@ -63,6 +64,7 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -80,22 +82,36 @@ from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
|
|||
|
||||
# Serious types first so column order is stable and self-documenting.
|
||||
ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
|
||||
# Output type axis = the 14 leaf types plus the two rollups, in that order.
|
||||
ROLLUP_TYPES: tuple[str, ...] = ("Serious crime", "Minor crime")
|
||||
ALL_OUTPUT_TYPES: tuple[str, ...] = ALL_CRIME_TYPES + ROLLUP_TYPES
|
||||
|
||||
DEFAULT_BUFFER_M = 100.0
|
||||
DEFAULT_BUFFER_M = 50.0
|
||||
MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
|
||||
STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
|
||||
|
||||
# Trailing-window definitions, in (label, years) form. Each window's average is
|
||||
# pooled over the force's covered months inside the window; one average-annual-
|
||||
# count column (`(/yr, <label>)`) — the filterable crime feature — is emitted per
|
||||
# window.
|
||||
WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))
|
||||
|
||||
# Per-incident records cover the longest window's calendar years so the
|
||||
# "individual crimes" list reconciles exactly with the headline counts: every
|
||||
# record is an incident inside that window and vice versa. The headline counts
|
||||
# are pooled per calendar year, so the records window must be calendar-year
|
||||
# aligned too -- a rolling month span (e.g. a fixed 84 months) would include
|
||||
# incidents from a year the headline excludes when the latest month is mid-year.
|
||||
RECORDS_WINDOW_YEARS = max(win_years for _, win_years in WINDOWS)
|
||||
|
||||
# Minimum covered months for a year to get a by-year chart bar (and to be
|
||||
# listed in `covered_years`). Annualising fewer observed months (x12 from a
|
||||
# single month at the worst) produces bars dominated by noise, and the first
|
||||
# (2010: one month) and current partial year would otherwise always chart as
|
||||
# spikes/dips. Six months keeps the annualisation factor <= 2.
|
||||
# single month at the worst) produces bars dominated by noise. Six months keeps
|
||||
# the annualisation factor <= 2.
|
||||
MIN_BAR_MONTHS = 6
|
||||
|
||||
# Forces that report nationwide rather than policing a territory. They never
|
||||
# define a postcode's home force (their publication calendar says nothing about
|
||||
# whether the *territorial* force covering the postcode published), but their
|
||||
# incidents still count toward whichever postcodes they fall in.
|
||||
# define a postcode's home force, but their incidents still count.
|
||||
NON_TERRITORIAL_FORCES = frozenset({"btp"})
|
||||
|
||||
COVERAGE_COLUMN = "covered_years"
|
||||
|
|
@ -106,8 +122,14 @@ LON_BOUNDS = (-9.5, 2.5)
|
|||
LAT_BOUNDS = (49.0, 61.5)
|
||||
|
||||
# Read CSVs in chunks of files to bound peak memory while keeping the STRtree
|
||||
# query vectorised over a useful number of points.
|
||||
_CSV_BATCH = 64
|
||||
# query vectorised over a useful number of points. Kept modest because the
|
||||
# in-window batches also materialise the per-incident record strings.
|
||||
_CSV_BATCH = 32
|
||||
|
||||
|
||||
def crime_column(type_name: str, window: str) -> str:
|
||||
"""Filterable average-annual-count column, e.g. ``"Burglary (/yr, 7y)"``."""
|
||||
return f"{type_name} (/yr, {window})"
|
||||
|
||||
|
||||
def _force_calendar(
|
||||
|
|
@ -115,12 +137,9 @@ def _force_calendar(
|
|||
) -> tuple[list[int], list[str], np.ndarray]:
|
||||
"""Derive the per-force publication calendar from the CSV paths.
|
||||
|
||||
Each police.uk file lives under ``{crime_dir}/{YYYY-MM}/{YYYY-MM}-{force}-
|
||||
street.csv`` and holds that force's incidents for that month, so file
|
||||
presence IS the coverage signal: a (force, month) with no file published
|
||||
nothing. Returns the sorted distinct years, the force slugs (sorted), and
|
||||
``months_in_year_force`` of shape (n_forces, n_years) -- how many months
|
||||
each force published in each year.
|
||||
File presence IS the coverage signal: a (force, month) with no file
|
||||
published nothing. Returns the sorted distinct years, the force slugs
|
||||
(sorted), and ``months_in_year_force`` of shape (n_forces, n_years).
|
||||
"""
|
||||
month_force: set[tuple[str, str]] = set()
|
||||
for path in csvs:
|
||||
|
|
@ -142,9 +161,6 @@ def _force_calendar(
|
|||
for month, force in month_force:
|
||||
months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1
|
||||
|
||||
# Surface coverage gaps loudly: any territorial force missing months inside
|
||||
# the global publication window is exactly the data hole the coverage
|
||||
# masking exists for.
|
||||
all_months = {month for month, _ in month_force}
|
||||
for force in forces:
|
||||
published = {m for m, f in month_force if f == force}
|
||||
|
|
@ -159,22 +175,25 @@ def _force_calendar(
|
|||
|
||||
def _build_tree(
|
||||
polygons: np.ndarray, buffer_m: float
|
||||
) -> tuple[np.ndarray, shapely.STRtree]:
|
||||
) -> tuple[shapely.STRtree, np.ndarray]:
|
||||
"""Buffer postcode polygons outward by ``buffer_m`` and index them.
|
||||
|
||||
Buffer index == postcode index. Geometries that fail to buffer are replaced
|
||||
with an empty polygon so the index stays aligned; they simply never match.
|
||||
Buffer index == postcode index. Returns the STRtree and a ``usable`` boolean
|
||||
mask: geometries that fail to buffer are replaced with an empty polygon (so
|
||||
the index stays aligned and they never match) and marked unusable, so their
|
||||
crime picture is reported as unknown rather than zero.
|
||||
"""
|
||||
buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
|
||||
broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
|
||||
if broken.any():
|
||||
print(f" {int(broken.sum()):,} postcode buffers unusable; left empty")
|
||||
buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
|
||||
return buffers, shapely.STRtree(buffers)
|
||||
return shapely.STRtree(buffers), ~broken
|
||||
|
||||
|
||||
def _accumulate_counts(
|
||||
csvs: list[Path],
|
||||
postcodes: np.ndarray,
|
||||
tree: shapely.STRtree,
|
||||
type_to_idx: dict[str, int],
|
||||
year_to_idx: dict[int, int],
|
||||
|
|
@ -182,35 +201,49 @@ def _accumulate_counts(
|
|||
transformer: Transformer,
|
||||
counts: np.ndarray,
|
||||
force_votes: np.ndarray,
|
||||
records_shard_dir: Path | None,
|
||||
records_min_ym: int,
|
||||
) -> None:
|
||||
"""Stream the crime CSVs, counting points-in-buffer per (postcode, type, year).
|
||||
|
||||
Also accumulates ``force_votes`` (n_postcodes, n_forces): how many matched
|
||||
incidents each force's files contributed to each postcode, which later
|
||||
elects the postcode's home force for the coverage calendar.
|
||||
Also accumulates ``force_votes`` (n_postcodes, n_forces) for the home-force
|
||||
election and, when ``records_shard_dir`` is set, writes one parquet shard per
|
||||
batch holding every counted (incident, postcode) pair whose month is within
|
||||
the records window (month index >= ``records_min_ym``).
|
||||
"""
|
||||
# Type overrides only for the columns we ever read; LSOA is not stored.
|
||||
schema = {
|
||||
"Longitude": pl.Float64,
|
||||
"Latitude": pl.Float64,
|
||||
"Month": pl.Utf8,
|
||||
"Crime type": pl.Utf8,
|
||||
"Location": pl.Utf8,
|
||||
"Last outcome category": pl.Utf8,
|
||||
}
|
||||
years = list(year_to_idx)
|
||||
total_points = 0
|
||||
total_matches = 0
|
||||
total_dropped = 0
|
||||
total_records = 0
|
||||
unknown_type_counts: dict[str, int] = {}
|
||||
|
||||
for start in range(0, len(csvs), _CSV_BATCH):
|
||||
batch = csvs[start : start + _CSV_BATCH]
|
||||
# The source file identifies the publishing force (police.uk has no
|
||||
# force column with consistent naming); map each path back to its
|
||||
# force index for the home-force vote.
|
||||
path_to_fidx = {}
|
||||
batch_max_ym = -1
|
||||
for path in batch:
|
||||
m = STREET_CSV_NAME_RE.fullmatch(path.name)
|
||||
if m is not None and m.group(2) in force_to_idx:
|
||||
path_to_fidx[str(path)] = force_to_idx[m.group(2)]
|
||||
if m is not None:
|
||||
ym = m.group(1)
|
||||
batch_max_ym = max(batch_max_ym, int(ym[:4]) * 12 + int(ym[5:7]) - 1)
|
||||
if m.group(2) in force_to_idx:
|
||||
path_to_fidx[str(path)] = force_to_idx[m.group(2)]
|
||||
# The per-incident record strings (Location, outcome) are by far the
|
||||
# heaviest columns; read them only for batches that fall inside the
|
||||
# records window, so the ~50% of pre-window months cost nothing extra.
|
||||
want_records = records_shard_dir is not None and batch_max_ym >= records_min_ym
|
||||
record_cols = ["Location", "Last outcome category"] if want_records else []
|
||||
|
||||
frame = (
|
||||
pl.scan_csv(
|
||||
batch,
|
||||
|
|
@ -218,12 +251,12 @@ def _accumulate_counts(
|
|||
ignore_errors=True,
|
||||
include_file_paths="_source_path",
|
||||
)
|
||||
.select("Longitude", "Latitude", "Month", "Crime type", "_source_path")
|
||||
# strict=False: a single malformed Month drops only that row instead
|
||||
# of aborting the whole build (a non-numeric year becomes null and is
|
||||
# filtered out by the year membership check below).
|
||||
.select(
|
||||
"Longitude", "Latitude", "Month", "Crime type", *record_cols, "_source_path"
|
||||
)
|
||||
.with_columns(
|
||||
pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year")
|
||||
pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year"),
|
||||
pl.col("Month").str.slice(5, 2).cast(pl.Int32, strict=False).alias("_mm"),
|
||||
)
|
||||
.filter(
|
||||
pl.col("Longitude").is_not_null()
|
||||
|
|
@ -234,14 +267,11 @@ def _accumulate_counts(
|
|||
& (pl.col("Crime type") != "")
|
||||
& pl.col("year").is_in(years)
|
||||
)
|
||||
# Canonicalise legacy pre-2014 crime-type names ("Violent crime",
|
||||
# "Public disorder and weapons") to their current equivalents before
|
||||
# indexing, so ~1.9M historical incidents are counted instead of
|
||||
# dropped. `.replace` leaves current types unchanged.
|
||||
# year*12 + (month-1): an integer month index for window filtering.
|
||||
.with_columns(
|
||||
(pl.col("year") * 12 + (pl.col("_mm").fill_null(1) - 1)).alias("month_index")
|
||||
)
|
||||
.with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
|
||||
# Map crime types to indices with default=None so an unrecognised
|
||||
# type yields a null index we can *report* rather than silently drop
|
||||
# (the legacy LSOA path surfaced unknown types via its dynamic pivot).
|
||||
.with_columns(
|
||||
pl.col("Crime type")
|
||||
.replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
|
||||
|
|
@ -253,7 +283,16 @@ def _accumulate_counts(
|
|||
.replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32)
|
||||
.alias("fidx"),
|
||||
)
|
||||
.select("Longitude", "Latitude", "Crime type", "tidx", "yidx", "fidx")
|
||||
.select(
|
||||
"Longitude",
|
||||
"Latitude",
|
||||
"Crime type",
|
||||
*record_cols,
|
||||
"month_index",
|
||||
"tidx",
|
||||
"yidx",
|
||||
"fidx",
|
||||
)
|
||||
.collect(engine="streaming")
|
||||
)
|
||||
|
||||
|
|
@ -273,19 +312,15 @@ def _accumulate_counts(
|
|||
tidx = frame["tidx"].to_numpy()
|
||||
yidx = frame["yidx"].to_numpy()
|
||||
fidx = frame["fidx"].to_numpy()
|
||||
month_index = frame["month_index"].to_numpy()
|
||||
|
||||
x, y = transformer.transform(lon, lat)
|
||||
finite = np.isfinite(x) & np.isfinite(y)
|
||||
total_dropped += int((~finite).sum())
|
||||
if not finite.any():
|
||||
continue
|
||||
x, y, tidx, yidx, fidx = (
|
||||
x[finite],
|
||||
y[finite],
|
||||
tidx[finite],
|
||||
yidx[finite],
|
||||
fidx[finite],
|
||||
)
|
||||
x, y = x[finite], y[finite]
|
||||
tidx, yidx, fidx = tidx[finite], yidx[finite], fidx[finite]
|
||||
total_points += x.size
|
||||
|
||||
points = shapely.points(x, y)
|
||||
|
|
@ -306,9 +341,26 @@ def _accumulate_counts(
|
|||
)
|
||||
total_matches += point_index.size
|
||||
|
||||
if want_records:
|
||||
total_records += _write_record_shard(
|
||||
records_shard_dir,
|
||||
start,
|
||||
postcodes,
|
||||
point_index,
|
||||
postcode_index,
|
||||
month_index[finite],
|
||||
frame["Crime type"].to_numpy()[finite],
|
||||
frame["Location"].to_numpy()[finite],
|
||||
frame["Last outcome category"].to_numpy()[finite],
|
||||
lon[finite],
|
||||
lat[finite],
|
||||
records_min_ym,
|
||||
)
|
||||
|
||||
print(
|
||||
f" files {start + len(batch):,}/{len(csvs):,}: "
|
||||
f"{total_points:,} located points, {total_matches:,} postcode matches"
|
||||
+ (f", {total_records:,} records" if records_shard_dir is not None else "")
|
||||
)
|
||||
|
||||
if total_dropped:
|
||||
|
|
@ -329,19 +381,65 @@ def _accumulate_counts(
|
|||
)
|
||||
|
||||
|
||||
def _write_record_shard(
|
||||
records_shard_dir: Path,
|
||||
start: int,
|
||||
postcodes: np.ndarray,
|
||||
point_index: np.ndarray,
|
||||
postcode_index: np.ndarray,
|
||||
month_index: np.ndarray,
|
||||
crime_type: np.ndarray,
|
||||
location: np.ndarray,
|
||||
outcome: np.ndarray,
|
||||
lon: np.ndarray,
|
||||
lat: np.ndarray,
|
||||
records_min_ym: int,
|
||||
) -> int:
|
||||
"""Write one parquet shard of (incident, postcode) records for this batch.
|
||||
|
||||
Each matched pair becomes a row -- the same multiplicity as the count -- so a
|
||||
postcode's records are exactly the incidents that made up its counts. Only
|
||||
incidents within the records window (month index >= ``records_min_ym``) are
|
||||
kept. Returns the number of rows written.
|
||||
"""
|
||||
mi = month_index[point_index]
|
||||
keep = mi >= records_min_ym
|
||||
if not keep.any():
|
||||
return 0
|
||||
pidx = point_index[keep]
|
||||
# Build the string columns from Python lists (.tolist()) rather than numpy
|
||||
# object arrays: an all-null Location/outcome slice is an all-None object
|
||||
# array that polars cannot cast to String, whereas a list of str|None infers
|
||||
# a nullable String column cleanly.
|
||||
shard = pl.DataFrame(
|
||||
{
|
||||
"postcode": postcodes[postcode_index[keep]].astype(str),
|
||||
"month_index": mi[keep].astype(np.int32),
|
||||
"crime_type": crime_type[pidx].astype(str),
|
||||
"location": location[pidx].tolist(),
|
||||
"outcome": outcome[pidx].tolist(),
|
||||
"lat": lat[pidx].astype(np.float32),
|
||||
"lon": lon[pidx].astype(np.float32),
|
||||
},
|
||||
schema_overrides={
|
||||
"postcode": pl.String,
|
||||
"crime_type": pl.String,
|
||||
"location": pl.String,
|
||||
"outcome": pl.String,
|
||||
},
|
||||
)
|
||||
shard.write_parquet(
|
||||
records_shard_dir / f"{start:08d}.parquet", compression="zstd"
|
||||
)
|
||||
return shard.height
|
||||
|
||||
|
||||
def _assign_home_force(
|
||||
postcodes: np.ndarray,
|
||||
force_votes: np.ndarray,
|
||||
forces: list[str],
|
||||
) -> np.ndarray:
|
||||
"""Elect each postcode's home (territorial) force.
|
||||
|
||||
Majority vote of matched incidents per publishing force; non-territorial
|
||||
forces (BTP) are excluded from the vote because their calendar says nothing
|
||||
about local coverage. Postcodes with no votes (no incidents ever, or
|
||||
BTP-only) inherit the majority force of their outcode, then the national
|
||||
modal force, so every postcode gets a coverage calendar.
|
||||
"""
|
||||
"""Elect each postcode's home (territorial) force by majority incident vote."""
|
||||
votes = force_votes.astype(np.int64, copy=True)
|
||||
for idx, force in enumerate(forces):
|
||||
if force in NON_TERRITORIAL_FORCES:
|
||||
|
|
@ -354,18 +452,15 @@ def _assign_home_force(
|
|||
if not has_vote.any():
|
||||
raise ValueError("No incidents matched any postcode; cannot assign forces")
|
||||
|
||||
# Outcode-majority fallback for postcodes with no (territorial) incidents.
|
||||
outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object)
|
||||
national_modal = int(
|
||||
np.bincount(home[has_vote], minlength=len(forces)).argmax()
|
||||
)
|
||||
national_modal = int(np.bincount(home[has_vote], minlength=len(forces)).argmax())
|
||||
if (~has_vote).any():
|
||||
outcode_modal: dict[str, int] = {}
|
||||
voted_outcodes = outcodes[has_vote]
|
||||
voted_home = home[has_vote]
|
||||
for oc in np.unique(voted_outcodes):
|
||||
counts = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
|
||||
outcode_modal[oc] = int(counts.argmax())
|
||||
tally = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
|
||||
outcode_modal[oc] = int(tally.argmax())
|
||||
fallback = np.array(
|
||||
[outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]],
|
||||
dtype=np.int32,
|
||||
|
|
@ -379,10 +474,82 @@ def _assign_home_force(
|
|||
return home
|
||||
|
||||
|
||||
def _window_annualised(
|
||||
counts: np.ndarray,
|
||||
months_in_year_force: np.ndarray,
|
||||
home_fidx: np.ndarray,
|
||||
usable: np.ndarray,
|
||||
year_mask: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Raw annualised incidents/yr per (postcode, type) over a window of years.
|
||||
|
||||
For each force, the count is pooled over the force's covered months that fall
|
||||
inside ``year_mask`` and annualised by those covered months. A covered
|
||||
postcode with no incidents of a type gets 0; a postcode whose force never
|
||||
published in the window, or whose geometry is unusable, gets NaN (unknown).
|
||||
"""
|
||||
n_pc, n_types = counts.shape[0], counts.shape[1]
|
||||
avg = np.full((n_pc, n_types), np.nan, dtype=np.float64)
|
||||
for f in range(months_in_year_force.shape[0]):
|
||||
sel = home_fidx == f
|
||||
if not sel.any():
|
||||
continue
|
||||
cov_months = months_in_year_force[f].astype(np.float64) * year_mask
|
||||
denom = cov_months.sum()
|
||||
if denom <= 0:
|
||||
continue # force published nothing in this window; stays NaN
|
||||
window_years = cov_months > 0
|
||||
pooled = counts[sel][:, :, window_years].sum(axis=2, dtype=np.float64)
|
||||
avg[sel] = pooled * 12.0 / denom
|
||||
avg[~usable] = np.nan
|
||||
return avg
|
||||
|
||||
|
||||
def _append_rollups(avg14: np.ndarray) -> np.ndarray:
|
||||
"""Append Serious/Minor rollup columns (sum of components) -> (n_pc, 16)."""
|
||||
serious_idx = [ALL_CRIME_TYPES.index(t) for t in SERIOUS_CRIME_TYPES]
|
||||
minor_idx = [ALL_CRIME_TYPES.index(t) for t in MINOR_CRIME_TYPES]
|
||||
serious = avg14[:, serious_idx].sum(axis=1)
|
||||
minor = avg14[:, minor_idx].sum(axis=1)
|
||||
return np.column_stack([avg14, serious, minor])
|
||||
|
||||
|
||||
def _write_crime_table(
|
||||
postcodes: np.ndarray,
|
||||
raw_by_window: dict[str, np.ndarray],
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Write the average-annual-count parquet (the filterable crime features).
|
||||
|
||||
``raw_by_window[label]`` is the (n_postcodes, 16) average annual incident
|
||||
count for that window (14 leaf types + 2 rollups). Each value is the raw,
|
||||
absolute incidents/yr; a postcode with no usable data for a window keeps NaN
|
||||
(written as null).
|
||||
"""
|
||||
data: dict[str, np.ndarray] = {"postcode": postcodes}
|
||||
for label, _years in WINDOWS:
|
||||
counts = np.round(raw_by_window[label], 1).astype(np.float32)
|
||||
for type_idx, name in enumerate(ALL_OUTPUT_TYPES):
|
||||
data[crime_column(name, label)] = counts[:, type_idx]
|
||||
|
||||
_write_nan_aware(data, output_path, "postcode crime average annual counts")
|
||||
|
||||
|
||||
def _write_nan_aware(
|
||||
data: dict[str, np.ndarray], output_path: Path, label: str
|
||||
) -> None:
|
||||
frame = pl.DataFrame(data)
|
||||
value_cols = [c for c in frame.columns if c != "postcode"]
|
||||
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
frame.write_parquet(output_path, compression="zstd")
|
||||
print(f"Wrote {label}: {output_path} {frame.shape}")
|
||||
|
||||
|
||||
def _rollup_long(
|
||||
long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
|
||||
) -> pl.DataFrame:
|
||||
"""Sum per-year annualised counts across ``types`` into a single rollup."""
|
||||
"""Sum per-year counts across ``types`` into a single rollup."""
|
||||
return (
|
||||
long.filter(pl.col("Crime type").is_in(list(types)))
|
||||
.group_by("postcode", "year")
|
||||
|
|
@ -392,108 +559,29 @@ def _rollup_long(
|
|||
)
|
||||
|
||||
|
||||
def _write_avg_yr(
|
||||
postcodes: np.ndarray,
|
||||
counts: np.ndarray,
|
||||
months_in_year_force: np.ndarray,
|
||||
home_fidx: np.ndarray,
|
||||
norm: np.ndarray,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Write ``postcode`` + ``"{type} (avg/yr)"`` density-normalised averages.
|
||||
|
||||
The headline is the POOLED annualised rate over the home force's covered
|
||||
months: ``sum(counts in covered years) * 12 / covered_months``. Years the
|
||||
force published nothing contribute neither incidents nor months, so a
|
||||
coverage gap (e.g. Greater Manchester 2019-07 onwards) is excluded instead
|
||||
of read as zero crime. Pooling over the full covered window -- rather than
|
||||
averaging only over years a type happened to occur -- is what keeps a
|
||||
single robbery-year from printing as a perennial robbery rate. Each
|
||||
postcode's value is then multiplied by ``norm`` (median_area / buffered
|
||||
catchment area) so the metric is a density rather than a footprint-inflated
|
||||
raw count; postcodes with unusable geometry (norm == 0) are null, not 0.
|
||||
"""
|
||||
n_postcodes, n_types = counts.shape[0], counts.shape[1]
|
||||
avg = np.full((n_postcodes, n_types), np.nan, dtype=np.float64)
|
||||
for f in range(months_in_year_force.shape[0]):
|
||||
sel = home_fidx == f
|
||||
if not sel.any():
|
||||
continue
|
||||
cov_months = months_in_year_force[f].astype(np.float64)
|
||||
denom = cov_months.sum()
|
||||
if denom <= 0:
|
||||
continue # force never published; stays null
|
||||
covered_years = cov_months > 0
|
||||
pooled = counts[sel][:, :, covered_years].sum(axis=2, dtype=np.float64)
|
||||
avg[sel] = pooled * 12.0 / denom
|
||||
|
||||
avg *= norm[:, None]
|
||||
avg[norm <= 0] = np.nan # unusable geometry: unknown, not zero
|
||||
avg = np.round(avg, 1).astype(np.float32)
|
||||
|
||||
data: dict[str, np.ndarray] = {"postcode": postcodes}
|
||||
for type_idx, name in enumerate(ALL_CRIME_TYPES):
|
||||
data[f"{name} (avg/yr)"] = avg[:, type_idx]
|
||||
|
||||
# Serious/Minor rollup headlines = the exact SUM of their component (avg/yr)
|
||||
# columns, so each rollup always equals the sum of the parts shown beside it
|
||||
# and can never fall below one of its own components. All components share
|
||||
# the postcode's pooled covered-month denominator, so the sum is itself the
|
||||
# pooled rollup rate. Null components (unusable geometry) propagate to a
|
||||
# null rollup.
|
||||
for rollup_name, rollup_types in (
|
||||
("Serious crime", SERIOUS_CRIME_TYPES),
|
||||
("Minor crime", MINOR_CRIME_TYPES),
|
||||
):
|
||||
rollup_idx = [ALL_CRIME_TYPES.index(name) for name in rollup_types]
|
||||
data[f"{rollup_name} (avg/yr)"] = np.round(
|
||||
avg[:, rollup_idx].sum(axis=1), 1
|
||||
).astype(np.float32)
|
||||
|
||||
frame = pl.DataFrame(data)
|
||||
value_cols = [c for c in frame.columns if c != "postcode"]
|
||||
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
frame.write_parquet(output_path, compression="zstd")
|
||||
print(f"Wrote postcode crime averages: {output_path}")
|
||||
|
||||
|
||||
def _write_by_year(
|
||||
postcodes: np.ndarray,
|
||||
counts: np.ndarray,
|
||||
years: list[int],
|
||||
months_in_year_force: np.ndarray,
|
||||
home_fidx: np.ndarray,
|
||||
norm: np.ndarray,
|
||||
usable: np.ndarray,
|
||||
min_bar_months: int,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Write nested ``"{type} (by year)"`` series plus rollups and coverage.
|
||||
"""Write nested ``"{type} (by year)"`` raw-count series plus rollups + coverage.
|
||||
|
||||
A bar is only emitted for (postcode, year)s where the postcode's home force
|
||||
published at least ``min_bar_months`` months -- annualising a thinner year
|
||||
(x12 from a single month at the extreme) charts noise, and a force-gap year
|
||||
must chart as *no data*, not zero. Bars are scaled by the force's covered
|
||||
months in that year and area-normalised by the same ``norm`` factor as the
|
||||
headline so chart and headline stay mutually consistent.
|
||||
|
||||
Every postcode gets a row (the output is dense) carrying ``covered_years``
|
||||
-- the list of {year, months} the home force published at least
|
||||
``min_bar_months`` months -- so consumers can distinguish covered-but-
|
||||
crime-free years (year listed, no bar => genuine zero) from coverage gaps
|
||||
(year absent => unknown). Postcodes with unusable geometry get an empty
|
||||
coverage list: their crime picture is unknown.
|
||||
published at least ``min_bar_months`` months. Bars are the raw annualised
|
||||
count for that year (``count * 12 / covered_months``); unlike the headline
|
||||
windows there is no per-capita normalisation here -- the chart shows incident
|
||||
volume over time. Every postcode gets a ``covered_years`` row so consumers can
|
||||
distinguish covered-but-crime-free years from coverage gaps.
|
||||
"""
|
||||
# (n_postcodes, n_years): covered months of each postcode's home force.
|
||||
cov_pc_year = months_in_year_force[home_fidx, :]
|
||||
usable = norm > 0
|
||||
|
||||
annual = np.round(
|
||||
counts.astype(np.float64)
|
||||
* 12.0
|
||||
/ np.maximum(cov_pc_year[:, None, :], 1)
|
||||
* norm[:, None, None],
|
||||
counts.astype(np.float64) * 12.0 / np.maximum(cov_pc_year[:, None, :], 1),
|
||||
1,
|
||||
)
|
||||
bar_ok = (
|
||||
|
|
@ -506,9 +594,6 @@ def _write_by_year(
|
|||
|
||||
type_names = np.array(ALL_CRIME_TYPES, dtype=object)
|
||||
year_values = np.array(years, dtype=np.int32)
|
||||
# Explicit schema: with full masking (e.g. every year below min_bar_months)
|
||||
# the fancy-indexed numpy object arrays are empty and polars would infer
|
||||
# Object columns, which breaks the rollup `is_in` below.
|
||||
long = pl.DataFrame(
|
||||
{
|
||||
"postcode": postcodes[pc_i].astype(str),
|
||||
|
|
@ -532,8 +617,6 @@ def _write_by_year(
|
|||
type_cols = [c for c in wide.columns if c != "postcode"]
|
||||
wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
|
||||
|
||||
# Dense base: every postcode, with its home force's coverage calendar.
|
||||
# Built per force (there are ~45) and joined on the force index.
|
||||
coverage_per_force: list[list[dict[str, int]]] = []
|
||||
for f in range(months_in_year_force.shape[0]):
|
||||
coverage_per_force.append(
|
||||
|
|
@ -562,7 +645,6 @@ def _write_by_year(
|
|||
dense = (
|
||||
base.join(coverage_frame, on="_fidx", how="left")
|
||||
.with_columns(
|
||||
# Unusable geometry: empty coverage -- the crime picture is unknown.
|
||||
pl.when(pl.col("_usable"))
|
||||
.then(pl.col(COVERAGE_COLUMN))
|
||||
.otherwise(pl.col(COVERAGE_COLUMN).list.head(0))
|
||||
|
|
@ -577,11 +659,47 @@ def _write_by_year(
|
|||
print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}")
|
||||
|
||||
|
||||
def _finalize_records(
|
||||
records_shard_dir: Path, output_path: Path
|
||||
) -> None:
|
||||
"""Concatenate the per-batch record shards into one postcode-sorted parquet.
|
||||
|
||||
Sorting by postcode lets the server build a contiguous per-postcode slice
|
||||
index. The sort runs on the streaming engine so it spills rather than holding
|
||||
all ~40M rows in memory.
|
||||
"""
|
||||
shards = sorted(records_shard_dir.glob("*.parquet"))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not shards:
|
||||
pl.DataFrame(
|
||||
schema={
|
||||
"postcode": pl.String,
|
||||
"month_index": pl.Int32,
|
||||
"crime_type": pl.String,
|
||||
"location": pl.String,
|
||||
"outcome": pl.String,
|
||||
"lat": pl.Float32,
|
||||
"lon": pl.Float32,
|
||||
}
|
||||
).write_parquet(output_path, compression="zstd")
|
||||
print(f"Wrote crime records: {output_path} (empty)")
|
||||
return
|
||||
|
||||
(
|
||||
pl.scan_parquet(shards)
|
||||
.sort("postcode")
|
||||
.sink_parquet(output_path, compression="zstd")
|
||||
)
|
||||
n = pl.scan_parquet(output_path).select(pl.len()).collect().item()
|
||||
print(f"Wrote crime records: {output_path} ({n:,} rows)")
|
||||
|
||||
|
||||
def transform_crime_spatial(
|
||||
crime_dir: Path,
|
||||
boundaries_dir: Path,
|
||||
output_path: Path,
|
||||
by_year_output_path: Path,
|
||||
records_output_path: Path,
|
||||
buffer_m: float = DEFAULT_BUFFER_M,
|
||||
max_postcodes: int | None = None,
|
||||
max_files: int | None = None,
|
||||
|
|
@ -594,6 +712,10 @@ def transform_crime_spatial(
|
|||
csvs = csvs[:max_files]
|
||||
|
||||
years, forces, months_in_year_force = _force_calendar(csvs)
|
||||
latest_year = years[-1]
|
||||
# Records cover the longest window's calendar years (January of its earliest
|
||||
# year onward), so they reconcile exactly with the calendar-year headline.
|
||||
records_min_ym = (latest_year - (RECORDS_WINDOW_YEARS - 1)) * 12
|
||||
print(
|
||||
f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces "
|
||||
f"({years[0]}-{years[-1]})"
|
||||
|
|
@ -601,27 +723,10 @@ def transform_crime_spatial(
|
|||
)
|
||||
|
||||
postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
|
||||
postcodes = np.asarray(postcodes)
|
||||
|
||||
print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
|
||||
buffers, tree = _build_tree(polygons, buffer_m)
|
||||
|
||||
# Area-normalisation factor (median_area / catchment_area): divides out the
|
||||
# size of each postcode's catchment so the count measures crime density, not
|
||||
# how much ground the buffer sweeps. We normalise by the *buffered* area --
|
||||
# the region that actually collects points -- rather than the raw polygon, so
|
||||
# a tiny unit postcode isn't over-inflated by the fixed buffer-ring floor.
|
||||
# Buffers are in EPSG:27700, so shapely.area is in m^2.
|
||||
areas = shapely.area(buffers).astype(np.float64)
|
||||
usable_area = np.isfinite(areas) & (areas > 0)
|
||||
if not usable_area.any():
|
||||
raise ValueError("No postcode buffers have a positive area to normalise by")
|
||||
median_area = float(np.median(areas[usable_area]))
|
||||
norm = np.zeros(len(postcodes), dtype=np.float64)
|
||||
norm[usable_area] = median_area / areas[usable_area]
|
||||
print(
|
||||
f"Area-normalising to median catchment area {median_area:,.0f} m^2 "
|
||||
f"({int(usable_area.sum()):,}/{len(areas):,} postcodes have usable area)"
|
||||
)
|
||||
tree, usable = _build_tree(polygons, buffer_m)
|
||||
|
||||
type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
|
||||
year_to_idx = {year: idx for idx, year in enumerate(years)}
|
||||
|
|
@ -630,25 +735,50 @@ def transform_crime_spatial(
|
|||
force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32)
|
||||
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
|
||||
_accumulate_counts(
|
||||
csvs, tree, type_to_idx, year_to_idx, force_to_idx, transformer, counts, force_votes
|
||||
)
|
||||
|
||||
home_fidx = _assign_home_force(np.asarray(postcodes), force_votes, forces)
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="crime_records_", dir=records_output_path.parent
|
||||
) as shard_dir_str:
|
||||
shard_dir = Path(shard_dir_str)
|
||||
_accumulate_counts(
|
||||
csvs,
|
||||
postcodes,
|
||||
tree,
|
||||
type_to_idx,
|
||||
year_to_idx,
|
||||
force_to_idx,
|
||||
transformer,
|
||||
counts,
|
||||
force_votes,
|
||||
shard_dir,
|
||||
records_min_ym,
|
||||
)
|
||||
|
||||
_write_avg_yr(
|
||||
postcodes, counts, months_in_year_force, home_fidx, norm, output_path
|
||||
)
|
||||
_write_by_year(
|
||||
postcodes,
|
||||
counts,
|
||||
years,
|
||||
months_in_year_force,
|
||||
home_fidx,
|
||||
norm,
|
||||
min_bar_months,
|
||||
by_year_output_path,
|
||||
)
|
||||
home_fidx = _assign_home_force(postcodes, force_votes, forces)
|
||||
|
||||
# Per-window raw annualised averages (14 leaf types + Serious/Minor).
|
||||
raw_by_window: dict[str, np.ndarray] = {}
|
||||
for label, win_years in WINDOWS:
|
||||
year_mask = np.array(
|
||||
[1.0 if y > latest_year - win_years else 0.0 for y in years]
|
||||
)
|
||||
avg14 = _window_annualised(
|
||||
counts, months_in_year_force, home_fidx, usable, year_mask
|
||||
)
|
||||
raw_by_window[label] = _append_rollups(avg14)
|
||||
|
||||
_write_crime_table(postcodes, raw_by_window, output_path)
|
||||
_write_by_year(
|
||||
postcodes,
|
||||
counts,
|
||||
years,
|
||||
months_in_year_force,
|
||||
home_fidx,
|
||||
usable,
|
||||
min_bar_months,
|
||||
by_year_output_path,
|
||||
)
|
||||
_finalize_records(shard_dir, records_output_path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
|
@ -671,7 +801,7 @@ def main() -> None:
|
|||
"--output",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output parquet: postcode + '{type} (avg/yr)' columns",
|
||||
help="Output parquet: postcode + '{type} (/yr, <window>)' average-annual-count columns",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-by-year",
|
||||
|
|
@ -680,10 +810,10 @@ def main() -> None:
|
|||
help="Output parquet: postcode + nested '{type} (by year)' columns",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buffer-m",
|
||||
type=float,
|
||||
default=DEFAULT_BUFFER_M,
|
||||
help="Outward buffer (metres) added to each postcode boundary",
|
||||
"--output-records",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output parquet: one row per counted incident (last 7 years), postcode-sorted",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-postcodes",
|
||||
|
|
@ -705,15 +835,12 @@ def main() -> None:
|
|||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.buffer_m <= 0:
|
||||
raise SystemExit("--buffer-m must be greater than zero")
|
||||
|
||||
transform_crime_spatial(
|
||||
crime_dir=args.input,
|
||||
boundaries_dir=args.boundaries,
|
||||
output_path=args.output,
|
||||
by_year_output_path=args.output_by_year,
|
||||
buffer_m=args.buffer_m,
|
||||
records_output_path=args.output_records,
|
||||
max_postcodes=args.max_postcodes,
|
||||
max_files=args.max_files,
|
||||
min_bar_months=args.min_bar_months,
|
||||
|
|
|
|||
149
pipeline/transform/join_price_estimates.py
Normal file
149
pipeline/transform/join_price_estimates.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
"""Join the slim price estimates back onto properties.parquet.
|
||||
|
||||
Price estimation runs on ``price_inputs.parquet`` (built by ``property_base``
|
||||
straight from epc_pp + arcgis, independently of merge's area features) and emits
|
||||
``price_estimates.parquet`` — the natural key (Postcode + coalesced address) plus
|
||||
``Estimated current price`` / ``Est. price per sqm``. This step joins those two
|
||||
columns onto properties.parquet to produce the file the server consumes.
|
||||
|
||||
Why the natural key
|
||||
-------------------
|
||||
Estimates and properties are built by separate runs, so a positional row index
|
||||
would not line up. Instead both derive the key ``(Postcode, coalesce(register
|
||||
address, EPC address))`` — which is unique and non-null on the deduped dwelling
|
||||
universe (see ``property_base._dedupe_collapsed_properties``) and identical on
|
||||
both sides because both start from that same universe. So estimates map onto
|
||||
properties 1:1 regardless of row order.
|
||||
|
||||
Re-running is safe: any pre-existing estimate columns are dropped first, and the
|
||||
join is keyed (not positional), so a second run reproduces the same result. The
|
||||
join refuses if any property has no estimate (the dwelling universes diverged,
|
||||
e.g. a stale price_inputs vs a newer epc_pp) rather than silently leaving prices
|
||||
null. Output is written to a temp file and atomically renamed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from pipeline.transform.price_estimation.utils import (
|
||||
ESTIMATE_COLUMNS,
|
||||
JOIN_ADDRESS,
|
||||
JOIN_KEYS,
|
||||
join_address_expr,
|
||||
)
|
||||
|
||||
|
||||
def join_estimates(properties: Path, estimates_path: Path) -> int:
|
||||
"""Augment ``properties`` in place with the estimate columns; return rows.
|
||||
|
||||
Joins the slim estimates onto properties by the natural key and atomically
|
||||
replaces properties.parquet. Idempotent: any estimate columns already on the
|
||||
file are dropped first.
|
||||
"""
|
||||
estimates = pl.scan_parquet(estimates_path)
|
||||
est_cols = estimates.collect_schema().names()
|
||||
missing = [c for c in (*JOIN_KEYS, *ESTIMATE_COLUMNS) if c not in est_cols]
|
||||
if missing:
|
||||
raise ValueError(f"{estimates_path}: missing columns {missing}")
|
||||
|
||||
stats = estimates.select(
|
||||
n=pl.len(), unique=pl.struct(JOIN_KEYS).n_unique()
|
||||
).collect(engine="streaming")
|
||||
n_estimates, n_unique = stats["n"][0], stats["unique"][0]
|
||||
if n_unique != n_estimates:
|
||||
raise ValueError(
|
||||
f"{estimates_path}: natural key {JOIN_KEYS} is not unique "
|
||||
f"({n_estimates - n_unique:,} duplicate rows)"
|
||||
)
|
||||
|
||||
n_properties = pl.scan_parquet(properties).select(pl.len()).collect().item()
|
||||
|
||||
# Drop any estimate columns already present (idempotent re-run) and attach the
|
||||
# coalesced-address half of the natural key.
|
||||
properties_keyed = (
|
||||
pl.scan_parquet(properties)
|
||||
.drop(ESTIMATE_COLUMNS, strict=False)
|
||||
.with_columns(join_address_expr())
|
||||
)
|
||||
|
||||
# Every property must have an estimate: estimates and properties come from the
|
||||
# same dwelling universe, so a gap means a stale/foreign price_inputs (e.g.
|
||||
# built from a different epc_pp). Fail loudly instead of nulling prices.
|
||||
#
|
||||
# This assumes properties.parquet contains ONLY epc_pp-derived dwellings, which
|
||||
# is true for the production merge output. Running merge with --actual-listings
|
||||
# appends listing seed rows whose (Postcode, address) keys are absent from
|
||||
# price_inputs (built straight from epc_pp), which would trip the guard below.
|
||||
# Enabling listing integration on the primary output therefore requires
|
||||
# price_inputs to include those seed rows too.
|
||||
unmatched = (
|
||||
properties_keyed.select(JOIN_KEYS)
|
||||
.join(estimates.select(JOIN_KEYS), on=JOIN_KEYS, how="anti")
|
||||
.select(pl.len())
|
||||
.collect(engine="streaming")
|
||||
.item()
|
||||
)
|
||||
if unmatched:
|
||||
raise ValueError(
|
||||
f"{properties}: {unmatched:,} of {n_properties:,} properties have no "
|
||||
"matching estimate; the price_inputs and properties dwelling universes "
|
||||
"differ (regenerate price_inputs.parquet from the current epc_pp)."
|
||||
)
|
||||
|
||||
# maintain_order="left" keeps properties in merge's row order; the unique key
|
||||
# cannot fan the join out, so the row count is preserved.
|
||||
result = properties_keyed.join(
|
||||
estimates, on=JOIN_KEYS, how="left", maintain_order="left"
|
||||
).drop(JOIN_ADDRESS)
|
||||
|
||||
tmp = properties.with_name(properties.name + ".tmp")
|
||||
result.sink_parquet(tmp)
|
||||
|
||||
written = pl.scan_parquet(tmp).select(pl.len()).collect().item()
|
||||
if written != n_properties:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise ValueError(
|
||||
f"{properties}: join changed the row count "
|
||||
f"({n_properties:,} -> {written:,})"
|
||||
)
|
||||
|
||||
tmp.replace(properties)
|
||||
return written
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Join price_estimates.parquet onto properties.parquet"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--properties",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="properties.parquet (read, then overwritten with the estimate "
|
||||
"columns joined in)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--estimates",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Slim price_estimates.parquet from price_estimation.estimate",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
written = join_estimates(args.properties, args.estimates)
|
||||
size_mb = args.properties.stat().st_size / (1024 * 1024)
|
||||
n_priced = (
|
||||
pl.scan_parquet(args.properties)
|
||||
.filter(pl.col("Estimated current price").is_not_null())
|
||||
.select(pl.len())
|
||||
.collect()
|
||||
.item()
|
||||
)
|
||||
print(f"Wrote {args.properties} ({size_mb:.1f} MB)")
|
||||
print(f" {written:,} rows, {n_priced:,} with an estimated current price")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -29,10 +29,16 @@ from pipeline.utils.fuzzy_join import (
|
|||
normalize_address_key,
|
||||
normalize_postcode_key,
|
||||
)
|
||||
from pipeline.transform.property_base import (
|
||||
MIN_FLOOR_AREA_M2,
|
||||
_active_english_postcode_area,
|
||||
_filter_to_active_english_postcodes,
|
||||
build_property_base,
|
||||
property_type_expr,
|
||||
)
|
||||
from pipeline.utils.normalize import drop_digit_tokens
|
||||
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
||||
|
||||
MIN_FLOOR_AREA_M2 = 10
|
||||
CONSERVATION_AREA_FEATURE = "Within conservation area"
|
||||
# Named "Tree canopy" (not "Street tree") because the underlying density unions
|
||||
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
|
||||
|
|
@ -77,23 +83,42 @@ _AREA_COLUMNS = [
|
|||
"% Mixed",
|
||||
"% White",
|
||||
"% Other",
|
||||
# Crime
|
||||
"Anti-social behaviour (avg/yr)",
|
||||
"Violence and sexual offences (avg/yr)",
|
||||
"Criminal damage and arson (avg/yr)",
|
||||
"Burglary (avg/yr)",
|
||||
"Vehicle crime (avg/yr)",
|
||||
"Robbery (avg/yr)",
|
||||
"Other theft (avg/yr)",
|
||||
"Shoplifting (avg/yr)",
|
||||
"Drugs (avg/yr)",
|
||||
"Possession of weapons (avg/yr)",
|
||||
"Public order (avg/yr)",
|
||||
"Bicycle theft (avg/yr)",
|
||||
"Theft from the person (avg/yr)",
|
||||
"Other crime (avg/yr)",
|
||||
"Serious crime (avg/yr)",
|
||||
"Minor crime (avg/yr)",
|
||||
# Crime — average annual recorded incident count (incidents/yr), 7-year and
|
||||
# 2-year windows. These are the filterable crime features; the per-incident
|
||||
# records live in a separate side table the server loads directly (it bypasses
|
||||
# the merge).
|
||||
"Anti-social behaviour (/yr, 7y)",
|
||||
"Anti-social behaviour (/yr, 2y)",
|
||||
"Violence and sexual offences (/yr, 7y)",
|
||||
"Violence and sexual offences (/yr, 2y)",
|
||||
"Criminal damage and arson (/yr, 7y)",
|
||||
"Criminal damage and arson (/yr, 2y)",
|
||||
"Burglary (/yr, 7y)",
|
||||
"Burglary (/yr, 2y)",
|
||||
"Vehicle crime (/yr, 7y)",
|
||||
"Vehicle crime (/yr, 2y)",
|
||||
"Robbery (/yr, 7y)",
|
||||
"Robbery (/yr, 2y)",
|
||||
"Other theft (/yr, 7y)",
|
||||
"Other theft (/yr, 2y)",
|
||||
"Shoplifting (/yr, 7y)",
|
||||
"Shoplifting (/yr, 2y)",
|
||||
"Drugs (/yr, 7y)",
|
||||
"Drugs (/yr, 2y)",
|
||||
"Possession of weapons (/yr, 7y)",
|
||||
"Possession of weapons (/yr, 2y)",
|
||||
"Public order (/yr, 7y)",
|
||||
"Public order (/yr, 2y)",
|
||||
"Bicycle theft (/yr, 7y)",
|
||||
"Bicycle theft (/yr, 2y)",
|
||||
"Theft from the person (/yr, 7y)",
|
||||
"Theft from the person (/yr, 2y)",
|
||||
"Other crime (/yr, 7y)",
|
||||
"Other crime (/yr, 2y)",
|
||||
"Serious crime (/yr, 7y)",
|
||||
"Serious crime (/yr, 2y)",
|
||||
"Minor crime (/yr, 7y)",
|
||||
"Minor crime (/yr, 2y)",
|
||||
# Amenities
|
||||
"Number of restaurants within 2km",
|
||||
"Number of grocery shops and supermarkets within 2km",
|
||||
|
|
@ -189,8 +214,6 @@ _FINAL_RENAME_COLUMNS = {
|
|||
"outstanding_primary_catchments": "Outstanding primary school catchments",
|
||||
"outstanding_secondary_catchments": "Outstanding secondary school catchments",
|
||||
"max_download_speed": "Max available download speed (Mbps)",
|
||||
"serious_crime_avg_yr": "Serious crime (avg/yr)",
|
||||
"minor_crime_avg_yr": "Minor crime (avg/yr)",
|
||||
"mean_monthly_rent": "Estimated monthly rent",
|
||||
"floor_height": "Interior height (m)",
|
||||
"was_council_house": "Former council house",
|
||||
|
|
@ -822,78 +845,6 @@ def _validate_property_postcodes(df: pl.DataFrame) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Return the supported postcode universe with geography join keys."""
|
||||
return (
|
||||
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
|
||||
.filter(pl.col("doterm").is_null())
|
||||
.select(
|
||||
pl.col("pcds").alias("postcode"),
|
||||
"lat",
|
||||
pl.col("long").alias("lon"),
|
||||
"ctry25cd",
|
||||
pl.col("lsoa21cd").alias("lsoa21"),
|
||||
pl.col("oa21cd").alias("oa21"),
|
||||
pl.col("pcon24cd").alias("pcon"),
|
||||
)
|
||||
.drop_nulls(["postcode"])
|
||||
.unique(["postcode"])
|
||||
)
|
||||
|
||||
|
||||
def _remap_terminated_postcodes(
|
||||
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return (
|
||||
wide.join(
|
||||
postcode_mapping,
|
||||
left_on="postcode",
|
||||
right_on="old_postcode",
|
||||
how="left",
|
||||
)
|
||||
.with_columns(
|
||||
pl.coalesce("new_postcode", "postcode").alias("postcode"),
|
||||
)
|
||||
.drop("new_postcode")
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Keep one row per (postcode, address) — the most-recent transaction.
|
||||
|
||||
The terminated-postcode remap can map two distinct postcodes onto one active
|
||||
successor, collapsing the same physical address onto a single
|
||||
(postcode, address) key with conflicting sale records. Keep the row with the
|
||||
latest date_of_transfer so the headline price/date reflect the most recent
|
||||
transaction; genuinely distinct addresses are untouched.
|
||||
|
||||
The dedup key coalesces the price-paid address with the EPC address: EPC-only
|
||||
dwellings (never sold) have a null pp_address, so keying on pp_address alone
|
||||
would collapse EVERY EPC-only dwelling in a postcode onto one
|
||||
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
|
||||
address is unique within its postcode (the EPC frame is deduped on
|
||||
address+postcode upstream), so the coalesced key keeps them distinct while
|
||||
leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
|
||||
a sale exists.
|
||||
"""
|
||||
return (
|
||||
wide.with_columns(
|
||||
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
|
||||
)
|
||||
.sort("date_of_transfer", descending=True, nulls_last=True)
|
||||
.unique(
|
||||
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
|
||||
)
|
||||
.drop("_dedupe_address")
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_active_english_postcodes(
|
||||
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return wide.join(active_postcodes, on="postcode", how="semi")
|
||||
|
||||
|
||||
def _join_area_side_tables(
|
||||
base: pl.LazyFrame,
|
||||
*,
|
||||
|
|
@ -923,21 +874,15 @@ def _join_area_side_tables(
|
|||
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
|
||||
base = base.join(tenure, on="lsoa21", how="left")
|
||||
|
||||
# Crime is counted spatially per postcode (incidents within 50m of the
|
||||
# postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
|
||||
# precomputes the Serious/Minor headline rollups as the mean of the by-year
|
||||
# rollup bars; read those straight through (renamed to the internal columns
|
||||
# _finalize_merged_columns expects) rather than re-summing the per-type
|
||||
# avg/yr columns — summing divides each type by its OWN years-present and
|
||||
# overstates the rollup when types differ in coverage. A postcode absent from
|
||||
# the crime table keeps null rollups via the left join (no fabricated zero);
|
||||
# the per-type avg/yr columns pass through unchanged for display.
|
||||
base = base.join(crime, on="postcode", how="left").rename(
|
||||
{
|
||||
"Serious crime (avg/yr)": "serious_crime_avg_yr",
|
||||
"Minor crime (avg/yr)": "minor_crime_avg_yr",
|
||||
}
|
||||
)
|
||||
# Crime is counted spatially per postcode (incidents within the boundary
|
||||
# buffer), so it joins on postcode rather than LSOA. crime_spatial writes
|
||||
# average-annual-count columns ("{type} (/yr, 7y|2y)"), including the
|
||||
# Serious/Minor rollups (the exact sum of their components); all pass straight
|
||||
# through to display/filtering. A postcode absent from the crime table keeps
|
||||
# null values via the left join (no fabricated zero). The per-incident records
|
||||
# are a separate side table the server loads directly, so it is not joined
|
||||
# here.
|
||||
base = base.join(crime, on="postcode", how="left")
|
||||
|
||||
base = base.join(median_age, on="lsoa21", how="left")
|
||||
base = base.join(election, on="pcon", how="left")
|
||||
|
|
@ -2386,27 +2331,17 @@ def _build(
|
|||
)
|
||||
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
||||
|
||||
wide = pl.scan_parquet(epc_pp_path).filter(
|
||||
pl.col("total_floor_area").is_null()
|
||||
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
|
||||
)
|
||||
|
||||
# Remap terminated postcodes to nearest active successor before filtering to
|
||||
# the supported active-English postcode universe. Historical properties from
|
||||
# terminated English postcodes are retained under their successor postcode.
|
||||
postcode_mapping = build_postcode_mapping(arcgis_path)
|
||||
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
|
||||
# The remap can collapse two terminated postcodes onto one active successor,
|
||||
# duplicating a physical address's (postcode, pp_address) key; keep only the
|
||||
# most-recent transaction per address before the per-postcode joins.
|
||||
wide = _dedupe_collapsed_properties(wide)
|
||||
# The dwelling universe — floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes — is shared with
|
||||
# price estimation so estimates line up 1:1 with these rows. See
|
||||
# pipeline.transform.property_base.
|
||||
wide = build_property_base(epc_pp_path, arcgis_path)
|
||||
arcgis_raw = pl.scan_parquet(arcgis_path)
|
||||
arcgis = _active_english_postcode_area(arcgis_raw)
|
||||
active_postcodes = arcgis.select("postcode").unique()
|
||||
active_postcode_count = (
|
||||
active_postcodes.select(pl.len()).collect(engine="streaming").item()
|
||||
)
|
||||
wide = _filter_to_active_english_postcodes(wide, active_postcodes)
|
||||
|
||||
if listed_buildings_path is not None:
|
||||
active_postcodes_for_listed = (
|
||||
|
|
@ -2542,37 +2477,10 @@ def _build(
|
|||
how="left",
|
||||
)
|
||||
|
||||
# Derive property_type: prefer EPC data, fall back to price-paid.
|
||||
# For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer detail.
|
||||
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
|
||||
["NO DATA!", "Not Recorded"]
|
||||
)
|
||||
has_epc = pl.col("epc_property_type").is_not_null()
|
||||
is_house = pl.col("epc_property_type") == "House"
|
||||
wide = wide.with_columns(
|
||||
pl.when(has_epc & is_house & ~bad_built_form)
|
||||
.then(pl.col("built_form"))
|
||||
.when(has_epc & is_house)
|
||||
.then(pl.col("pp_property_type"))
|
||||
.when(has_epc)
|
||||
.then(pl.col("epc_property_type"))
|
||||
.otherwise(pl.col("pp_property_type"))
|
||||
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
|
||||
# collapse terrace sub-types, and fold rare types into "Other"
|
||||
.replace(
|
||||
{
|
||||
"Flat": "Flats/Maisonettes",
|
||||
"Maisonette": "Flats/Maisonettes",
|
||||
"End-Terrace": "Terraced",
|
||||
"Mid-Terrace": "Terraced",
|
||||
"Enclosed End-Terrace": "Terraced",
|
||||
"Enclosed Mid-Terrace": "Terraced",
|
||||
"Bungalow": "Other",
|
||||
"Park home": "Other",
|
||||
}
|
||||
)
|
||||
.alias("property_type")
|
||||
)
|
||||
# Derive property_type (EPC preferred, price-paid fallback, built_form for
|
||||
# houses). Shared with price_inputs so the estimate uses the same type; see
|
||||
# property_base.property_type_expr.
|
||||
wide = wide.with_columns(property_type_expr().alias("property_type"))
|
||||
|
||||
wide = wide.with_columns(
|
||||
pl.when(pl.col("duration") == "U")
|
||||
|
|
|
|||
|
|
@ -79,7 +79,9 @@ The output of `process_oa` is `list[(postcode, polygon)]` — the per-OA fragmen
|
|||
|
||||
**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 3000–5000 m² building blocks) before this change.
|
||||
|
||||
**GeoJSON output** (`output.py:write_district_geojson`): Two passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
|
||||
**Greenspace subtraction is connectivity-preserving** (`greenspace.py:subtract_greenspace`): park/water polygons are subtracted from each postcode, but greenspace that *crosses* a postcode (a river, a strip of parkland, a golf course through a village) would otherwise split it into scattered pieces. When the subtraction disconnects a postcode, `_reconnect_split` re-adds the narrowest removed necks — a morphological closing (`_RECONNECT_BRIDGE_M`, 25 m) clipped to the original postcode footprint — so parts ≤ ~50 m apart stay joined by a thin bridge of the postcode's own land (no address moves); genuinely wide barriers stay subtracted and the postcode legitimately splits.
|
||||
|
||||
**GeoJSON output** (`output.py:write_district_geojson`): three passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 **de-fragments** the partition (`_eliminate_small_detached_parts`): a detached part that is *both* small in absolute terms (< `_ELIM_ABS_MAX_M2`, 3000 m²) *and* a minor fraction (< `_ELIM_FRAC_MAX`, 15%) of its postcode is absorbed into the neighbouring postcode it shares the most boundary with — the classic GIS *eliminate*. This removes the Voronoi/INSPIRE/seam *scatter* that left ~1/3 of postcodes non-contiguous, while a genuine bisection (two substantial parts split by a river/railway) keeps both parts. The land is **reassigned**, never dropped, so the output stays a gapless partition and coverage is conserved; the largest part of every postcode is always retained, so no active postcode is dropped (a tiny neighbour-less sliver in removed greenspace is dropped, a larger isolated patch is kept). Pass 3 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
|
||||
|
||||
## Memory architecture
|
||||
|
||||
|
|
@ -107,6 +109,7 @@ Key design choices:
|
|||
2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
|
||||
3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
|
||||
4. **A postcode split across an OA seam keeps all its substantial parts** — `merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
|
||||
5. **Postcodes are contiguous unless genuinely split** — most non-contiguity is *scatter* (a unit drawn as many disconnected specks) from point-Voronoi over sparse/interleaved UPRNs, greenspace cutting across a unit, and overlap/seam slivers. Connectivity-preserving greenspace subtraction + the `_eliminate_small_detached_parts` de-fragmentation pass absorb that scatter into neighbours (coverage-conserving), cutting the share of multi-part postcodes roughly in half (~30% → ~14% measured on the worst rural/coastal districts) without dropping any postcode or leaving coverage gaps. Genuine bisections (river/railway/major road, or a detached part above the absolute+fraction thresholds) are preserved.
|
||||
|
||||
## Module structure
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from shapely import make_valid, wkb
|
|||
from shapely.geometry import MultiPolygon, Polygon
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
from .geometry import safe_difference, safe_union
|
||||
from .geometry import _SNAP_GRID, _poly_valid, safe_difference, safe_intersection, safe_union
|
||||
|
||||
|
||||
def load_greenspace(path: Path) -> tuple[STRtree, list]:
|
||||
|
|
@ -36,6 +36,42 @@ def load_greenspace(path: Path) -> tuple[STRtree, list]:
|
|||
|
||||
MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed
|
||||
|
||||
# Greenspace that merely trims the edge of a postcode is fine, but greenspace
|
||||
# that CROSSES it (a river, a strip of parkland, a golf course running through a
|
||||
# village) splits the postcode into a MultiPolygon -- one the map then draws as
|
||||
# several disconnected pieces. When subtraction disconnects a postcode, re-add
|
||||
# the postcode's OWN removed land along the narrowest necks (a morphological
|
||||
# closing clipped to the original footprint) so the parts stay joined by a thin
|
||||
# bridge. Parts left more than ~2x this width apart (a genuinely wide barrier)
|
||||
# stay split. Because the bridge is the postcode's own land, no address moves and
|
||||
# only a thin sliver of green is kept back.
|
||||
_RECONNECT_BRIDGE_M = 25.0
|
||||
|
||||
|
||||
def _reconnect_split(
|
||||
result: Polygon | MultiPolygon, postcode_geom: Polygon | MultiPolygon
|
||||
) -> Polygon | MultiPolygon:
|
||||
"""Re-join postcode parts that greenspace subtraction pulled apart by re-adding
|
||||
the narrow removed necks (within the original postcode), leaving wide barriers
|
||||
intact."""
|
||||
if result.geom_type != "MultiPolygon":
|
||||
return result
|
||||
closed = result.buffer(_RECONNECT_BRIDGE_M).buffer(-_RECONNECT_BRIDGE_M)
|
||||
if not closed.is_valid:
|
||||
closed = make_valid(closed)
|
||||
# The closing material that lies inside the original postcode but outside the
|
||||
# subtraction result == the thin green necks linking the parts. The exact
|
||||
# overlay path can leave line/point debris (coincident edges) that is
|
||||
# zero-area but NOT is_empty; `_poly_valid` strips it to polygons only, so the
|
||||
# is_empty guard works and the union can never return a GeometryCollection
|
||||
# (which `to_wgs84_geojson_multi` would silently truncate to a single part).
|
||||
bridges = _poly_valid(
|
||||
safe_difference(safe_intersection(closed, postcode_geom), result), _SNAP_GRID
|
||||
)
|
||||
if bridges.is_empty:
|
||||
return result
|
||||
return _poly_valid(safe_union([result, bridges]), _SNAP_GRID)
|
||||
|
||||
|
||||
def subtract_greenspace(
|
||||
postcode_geom: Polygon | MultiPolygon,
|
||||
|
|
@ -48,6 +84,10 @@ def subtract_greenspace(
|
|||
of intersecting greenspace from the postcode polygon. If subtraction
|
||||
would remove >90% of the area, keeps the original (the postcode
|
||||
genuinely covers that land, e.g. churchyards, riverside addresses).
|
||||
|
||||
If the subtraction disconnects the postcode (greenspace crossing it),
|
||||
:func:`_reconnect_split` re-adds the narrowest removed necks so the postcode
|
||||
stays a single piece rather than shipping as scattered fragments.
|
||||
"""
|
||||
candidate_idxs = tree.query(postcode_geom)
|
||||
if len(candidate_idxs) == 0:
|
||||
|
|
@ -74,4 +114,4 @@ def subtract_greenspace(
|
|||
if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION):
|
||||
return postcode_geom
|
||||
|
||||
return result
|
||||
return _reconnect_split(result, postcode_geom)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import math
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
|
@ -383,6 +384,164 @@ def _resolve_overlaps(
|
|||
return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
|
||||
|
||||
|
||||
# A detached (non-largest) part of a postcode is absorbed into the neighbouring
|
||||
# postcode it shares the most boundary with, when the part is BOTH small in
|
||||
# absolute terms AND a minor fraction of its postcode. This removes the
|
||||
# Voronoi/INSPIRE/overlap-seam SCATTER that otherwise left ~1/3 of postcodes
|
||||
# non-contiguous (a single unit drawn as many disconnected specks), while keeping
|
||||
# genuine splits: a postcode bisected by a river/railway into two SUBSTANTIAL
|
||||
# parts has both parts above these thresholds and is left alone. The land is
|
||||
# REASSIGNED to an adjacent postcode wherever a neighbour exists, so coverage is
|
||||
# conserved and the output stays a partition; only a tiny neighbour-less sliver
|
||||
# (< _ELIM_DROP_BELOW_M2, snap/overlap debris floating in removed greenspace) is
|
||||
# dropped, and a larger isolated part (a genuine detached hamlet) is kept. The
|
||||
# largest part of every postcode is always retained, so no postcode is dropped.
|
||||
_ELIM_ABS_MAX_M2 = 3000.0 # absolute size below which a minor part reads as scatter
|
||||
_ELIM_FRAC_MAX = 0.15 # ...and only when it is < this fraction of the postcode
|
||||
_ELIM_DROP_BELOW_M2 = 200.0 # a neighbour-less sliver this small is snap debris -> drop
|
||||
# WGS84-degree distances at UK latitudes (1e-6 deg ~ 0.11 m at ~53N)
|
||||
_ELIM_SHARED_EPS_DEG = 5e-7 # ~0.05 m: thin probe whose overlap area ~ shared-edge length
|
||||
# Candidate-gather + nearest-neighbour fallback radius, in degrees so it needs no
|
||||
# per-part reprojection. The metric reach is mildly anisotropic (~2.2 m N-S,
|
||||
# ~1.3-1.4 m E-W at England's latitudes); this only bounds the rare gapped-seam
|
||||
# fallback (the dominant border-sharing path is unaffected), so the approximation
|
||||
# is harmless. Gather and accept use the SAME radius, so there is no dead band.
|
||||
_ELIM_NEAREST_MAX_DEG = 2e-5
|
||||
_M_PER_DEG_LAT = 111_320.0
|
||||
_ELIM_ITERATIONS = 2
|
||||
|
||||
|
||||
def _approx_area_m2(geom_deg) -> float:
|
||||
"""Metric area of a WGS84 polygon via a latitude-scaled planar approximation.
|
||||
|
||||
Accurate to a few percent at the few-hundred-to-few-thousand m^2 scale these
|
||||
thresholds work at, and far cheaper than a per-part CRS transform over the
|
||||
full ~1.8M postcodes.
|
||||
"""
|
||||
area_deg2 = geom_deg.area
|
||||
if area_deg2 == 0.0:
|
||||
return 0.0
|
||||
centroid = geom_deg.centroid
|
||||
if centroid.is_empty:
|
||||
lat = (geom_deg.bounds[1] + geom_deg.bounds[3]) / 2
|
||||
else:
|
||||
lat = centroid.y
|
||||
return area_deg2 * _M_PER_DEG_LAT * _M_PER_DEG_LAT * math.cos(math.radians(lat))
|
||||
|
||||
|
||||
def _geom_parts(geom):
|
||||
return list(geom.geoms) if geom.geom_type == "MultiPolygon" else [geom]
|
||||
|
||||
|
||||
def _eliminate_small_detached_parts(
|
||||
items: list[tuple[str, Polygon | MultiPolygon]],
|
||||
) -> list[tuple[str, Polygon | MultiPolygon]]:
|
||||
"""De-fragment the partition: absorb small detached parts into the neighbour
|
||||
they border most (see the module note above).
|
||||
|
||||
Runs on the de-overlapped WGS84 geometries as the last shaping step. The
|
||||
largest part of every postcode is always retained, so no active postcode is
|
||||
dropped; parts are moved between postcodes (or, for tiny neighbour-less
|
||||
slivers, dropped), so total covered area is conserved to within rounding.
|
||||
"""
|
||||
geoms: dict[str, Polygon | MultiPolygon] = {pc: g for pc, g in items}
|
||||
for _ in range(_ELIM_ITERATIONS):
|
||||
recs: list[tuple[str, Polygon, float]] = []
|
||||
total: dict[str, float] = defaultdict(float)
|
||||
largest: dict[str, float] = defaultdict(float)
|
||||
for pc, geom in geoms.items():
|
||||
for part in _geom_parts(geom):
|
||||
if part.is_empty:
|
||||
continue
|
||||
area = _approx_area_m2(part)
|
||||
recs.append((pc, part, area))
|
||||
total[pc] += area
|
||||
if area > largest[pc]:
|
||||
largest[pc] = area
|
||||
|
||||
tree = STRtree([part for _, part, _ in recs])
|
||||
assign: dict[int, str | None] = {}
|
||||
for i, (pc, part, area) in enumerate(recs):
|
||||
if area >= largest[pc]:
|
||||
continue # the largest part always stays -> a postcode never vanishes
|
||||
if not (area < _ELIM_ABS_MAX_M2 and area < _ELIM_FRAC_MAX * total[pc]):
|
||||
continue # substantial or major-fraction part = genuine split, keep
|
||||
|
||||
best_pc = None
|
||||
best_score = 0.0
|
||||
nearest_pc = None
|
||||
nearest_d = float("inf")
|
||||
# Gather candidates out to the nearest-fallback radius (not just the
|
||||
# smaller border-probe radius), so a gapped seam in the
|
||||
# (border, nearest] band can actually be reassigned rather than dropped.
|
||||
for j in tree.query(part.buffer(_ELIM_NEAREST_MAX_DEG)):
|
||||
if j == i:
|
||||
continue
|
||||
other_pc, other_geom, _ = recs[j]
|
||||
if other_pc == pc:
|
||||
continue
|
||||
try:
|
||||
score = part.buffer(_ELIM_SHARED_EPS_DEG).intersection(other_geom).area
|
||||
except GEOSException:
|
||||
score = 0.0
|
||||
# Ties broken by the lexicographically smaller postcode so the
|
||||
# result is independent of STRtree traversal order (matches the
|
||||
# stable tie-break in _resolve_overlaps; keeps output byte-stable
|
||||
# across shapely/GEOS upgrades for content-hash caching).
|
||||
if score > best_score or (
|
||||
score == best_score and best_pc is not None and other_pc < best_pc
|
||||
):
|
||||
best_score = score
|
||||
best_pc = other_pc
|
||||
dist = part.distance(other_geom)
|
||||
if dist < nearest_d or (
|
||||
dist == nearest_d
|
||||
and nearest_pc is not None
|
||||
and other_pc < nearest_pc
|
||||
):
|
||||
nearest_d = dist
|
||||
nearest_pc = other_pc
|
||||
|
||||
if best_pc is not None and best_score > 0:
|
||||
assign[i] = best_pc # shares a border -> absorb into that neighbour
|
||||
elif nearest_pc is not None and nearest_d <= _ELIM_NEAREST_MAX_DEG:
|
||||
assign[i] = nearest_pc # gapped seam -> nearest neighbour
|
||||
elif area < _ELIM_DROP_BELOW_M2:
|
||||
assign[i] = None # neighbour-less snap/overlap sliver -> drop
|
||||
# else: keep with its own postcode (a genuine isolated patch)
|
||||
|
||||
if not assign:
|
||||
break
|
||||
|
||||
new_parts: dict[str, list] = defaultdict(list)
|
||||
for i, (pc, part, _) in enumerate(recs):
|
||||
if i in assign:
|
||||
target = assign[i]
|
||||
if target is None:
|
||||
continue
|
||||
new_parts[target].append(part)
|
||||
else:
|
||||
new_parts[pc].append(part)
|
||||
|
||||
rebuilt: dict[str, Polygon | MultiPolygon] = {}
|
||||
for pc, parts in new_parts.items():
|
||||
if len(parts) == 1:
|
||||
rebuilt[pc] = parts[0]
|
||||
else:
|
||||
merged = safe_union(parts, grid=_OUTPUT_PRECISION_DEG)
|
||||
if not merged.is_empty:
|
||||
rebuilt[pc] = merged
|
||||
# Carry forward any postcode that contributed no parts to recs (e.g. an
|
||||
# all-empty input geometry, which the part loop skips): never drop a
|
||||
# postcode just because reassignment fired elsewhere in this pass.
|
||||
for pc, geom in geoms.items():
|
||||
if pc not in rebuilt:
|
||||
rebuilt[pc] = geom
|
||||
geoms = rebuilt
|
||||
|
||||
return list(geoms.items())
|
||||
|
||||
|
||||
def _round_coords(coords, ndigits=6):
|
||||
if coords and isinstance(coords[0], (int, float)):
|
||||
return [round(coords[0], ndigits), round(coords[1], ndigits)]
|
||||
|
|
@ -501,6 +660,11 @@ def write_district_geojson(
|
|||
# Remove overlap strips so the output is a clean partition.
|
||||
converted = _resolve_overlaps(converted)
|
||||
|
||||
# De-fragment: absorb small detached parts (Voronoi/INSPIRE/seam scatter) into
|
||||
# the neighbour they border most, so postcodes stop shipping as many
|
||||
# disconnected specks. Coverage-preserving; runs on the final partition.
|
||||
converted = _eliminate_small_detached_parts(converted)
|
||||
|
||||
by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
|
||||
for pc, geom in converted:
|
||||
parts = pc.split()
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Each test targets a specific bug or edge case identified during code review.
|
|||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
|
@ -21,6 +22,8 @@ from .inspire import build_inspire_index
|
|||
from .oa_boundaries import parse_gpkg_geometry
|
||||
from .greenspace import subtract_greenspace
|
||||
from .output import (
|
||||
_approx_area_m2,
|
||||
_eliminate_small_detached_parts,
|
||||
_fill_holes,
|
||||
merge_fragments,
|
||||
to_wgs84_geojson,
|
||||
|
|
@ -1830,3 +1833,231 @@ class TestFragmentsCache:
|
|||
cache.write_text("c")
|
||||
# arcgis is optional/absent — it cannot have invalidated the cache.
|
||||
assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# De-fragmentation: small detached parts are absorbed into their best neighbour
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# WGS84 helper: build a box from METRE offsets around a fixed England anchor, so
|
||||
# the eliminate pass (which works in WGS84 degrees and measures area with a
|
||||
# latitude-scaled approximation) sees realistic coordinates and areas.
|
||||
_ANCHOR_LON, _ANCHOR_LAT = -0.1, 51.5
|
||||
_M_PER_DEG_LAT = 111_320.0
|
||||
_M_PER_DEG_LON = 111_320.0 * math.cos(math.radians(_ANCHOR_LAT))
|
||||
|
||||
|
||||
def _mbox(x0, y0, x1, y1):
|
||||
return box(
|
||||
_ANCHOR_LON + x0 / _M_PER_DEG_LON,
|
||||
_ANCHOR_LAT + y0 / _M_PER_DEG_LAT,
|
||||
_ANCHOR_LON + x1 / _M_PER_DEG_LON,
|
||||
_ANCHOR_LAT + y1 / _M_PER_DEG_LAT,
|
||||
)
|
||||
|
||||
|
||||
def _as_dict(items):
|
||||
return dict(items)
|
||||
|
||||
|
||||
class TestEliminateSmallDetachedParts:
|
||||
"""A small detached part should be absorbed into the postcode it borders most,
|
||||
de-fragmenting the unit while conserving coverage and never dropping a
|
||||
postcode. Genuine large splits must survive."""
|
||||
|
||||
def test_small_island_absorbed_by_surrounding_neighbour(self):
|
||||
# A: a big main blob (left) plus a small 30x30m island sitting INSIDE B's
|
||||
# territory; B fills the middle/right with a hole where the island is.
|
||||
main_a = _mbox(0, 0, 100, 200) # 20000 m²
|
||||
island = _mbox(150, 90, 180, 120) # 900 m², surrounded by B
|
||||
a = MultiPolygon([main_a, island])
|
||||
b = _mbox(100, 0, 300, 200).difference(island) # hole around the island
|
||||
before = unary_union([a, b]).area
|
||||
|
||||
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
|
||||
|
||||
assert "A" in out and "B" in out, "no postcode may be dropped"
|
||||
assert out["A"].geom_type == "Polygon", "A's island should be absorbed away"
|
||||
# The island area moves to B (the surrounding postcode); coverage is kept.
|
||||
assert out["B"].contains(island.representative_point())
|
||||
after = unary_union([out["A"], out["B"]]).area
|
||||
assert after == pytest.approx(before, rel=1e-6), "coverage must be conserved"
|
||||
|
||||
def test_genuine_large_split_is_kept(self):
|
||||
# Two substantial parts (both 40000 m²) far apart — a real bisection, not
|
||||
# scatter. Neither is below the absolute/fraction thresholds, so both stay.
|
||||
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(400, 0, 600, 200)])
|
||||
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||
assert out["A"].geom_type == "MultiPolygon"
|
||||
assert len(out["A"].geoms) == 2
|
||||
|
||||
def test_midsize_minor_part_kept_when_above_abs_threshold(self):
|
||||
# A big main plus a 10000 m² detached part: above the 3000 m² absolute
|
||||
# threshold, so it is a genuine secondary piece and must NOT be eliminated,
|
||||
# even though it is a small fraction of the postcode.
|
||||
a = MultiPolygon([_mbox(0, 0, 300, 300), _mbox(1000, 0, 1100, 100)])
|
||||
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||
assert out["A"].geom_type == "MultiPolygon"
|
||||
|
||||
def test_tiny_neighbourless_sliver_is_dropped(self):
|
||||
# A main blob plus a 10x10m (100 m²) sliver far from anything: below the
|
||||
# drop threshold with no neighbour to absorb it -> dropped as snap debris.
|
||||
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(1000, 1000, 1010, 1010)])
|
||||
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||
assert "A" in out, "the postcode itself must survive"
|
||||
assert out["A"].geom_type == "Polygon"
|
||||
|
||||
def test_largest_part_always_retained_no_postcode_dropped(self):
|
||||
# Even a postcode that is ENTIRELY a tiny sliver keeps its (largest) part —
|
||||
# active postcodes must never be dropped (validate_outputs is zero-tolerance).
|
||||
tiny = _mbox(500, 500, 505, 505) # 25 m², the whole postcode
|
||||
big = _mbox(0, 0, 300, 300)
|
||||
out = _as_dict(
|
||||
_eliminate_small_detached_parts([("TINY", tiny), ("BIG", big)])
|
||||
)
|
||||
assert "TINY" in out and not out["TINY"].is_empty
|
||||
assert "BIG" in out
|
||||
|
||||
def test_no_overlaps_introduced(self):
|
||||
# Absorbing parts must keep the output a partition (no double-coverage).
|
||||
main_a = _mbox(0, 0, 100, 200)
|
||||
island = _mbox(150, 90, 180, 120)
|
||||
a = MultiPolygon([main_a, island])
|
||||
b = _mbox(100, 0, 300, 200).difference(island)
|
||||
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
|
||||
overlap = out["A"].intersection(out["B"]).area
|
||||
assert overlap < (1.0 / (_M_PER_DEG_LAT * _M_PER_DEG_LON)), "no overlap"
|
||||
|
||||
def test_empty_postcode_not_dropped_when_reassignment_fires(self):
|
||||
# Regression: an empty-geometry postcode must survive even when a
|
||||
# reassignment fires elsewhere in the same pass (the rebuild used to drop
|
||||
# any key absent from `recs`). No active postcode may ever be dropped.
|
||||
main_a = _mbox(0, 0, 100, 200)
|
||||
island = _mbox(150, 90, 180, 120) # B absorbs this -> reassignment fires
|
||||
a = MultiPolygon([main_a, island])
|
||||
b = _mbox(100, 0, 300, 200).difference(island)
|
||||
empty = Polygon() # degenerate input that contributes no parts
|
||||
out = _as_dict(
|
||||
_eliminate_small_detached_parts([("A", a), ("B", b), ("EMPTY", empty)])
|
||||
)
|
||||
assert "EMPTY" in out, "an empty-geometry postcode must not be dropped"
|
||||
assert "A" in out and "B" in out
|
||||
|
||||
def test_tiebreak_is_deterministic_smaller_postcode_wins(self):
|
||||
# An island bordered by TWO postcodes with EQUAL shared edges must go to
|
||||
# the lexicographically smaller postcode, independent of STRtree order.
|
||||
a_main = _mbox(0, 0, 100, 100) # A's main, far from the island
|
||||
island = _mbox(300, 50, 320, 70) # 400 m², A's detached part
|
||||
a = MultiPolygon([a_main, island])
|
||||
bbb = _mbox(250, 0, 300, 120) # borders island's left edge
|
||||
ccc = _mbox(320, 0, 370, 120) # borders island's right edge (equal)
|
||||
out = _as_dict(
|
||||
_eliminate_small_detached_parts([("A", a), ("BBB", bbb), ("CCC", ccc)])
|
||||
)
|
||||
pt = island.representative_point()
|
||||
assert out["BBB"].contains(pt), "tie must go to the smaller postcode string"
|
||||
assert not out["CCC"].contains(pt)
|
||||
assert out["A"].geom_type == "Polygon"
|
||||
|
||||
def test_gapped_sliver_within_nearest_radius_is_reassigned_not_dropped(self):
|
||||
# A 300 m² sliver 1.2 m from its only neighbour N — beyond the border probe
|
||||
# but inside the nearest-fallback radius. Before the gather buffer was
|
||||
# widened to the accept radius, the old (smaller) gather buffer never
|
||||
# returned N, so the sliver fell through to the drop branch. Now it is
|
||||
# reassigned to N, conserving coverage.
|
||||
x_main = _mbox(0, 0, 100, 100) # far from the sliver
|
||||
x_sliver = _mbox(278.8, 50, 298.8, 65) # 300 m², right edge at x=298.8
|
||||
x = MultiPolygon([x_main, x_sliver])
|
||||
n = _mbox(300, 0, 400, 200) # left edge at x=300 -> 1.2 m gap
|
||||
before = unary_union([x, n]).area
|
||||
out = _as_dict(_eliminate_small_detached_parts([("X", x), ("N", n)]))
|
||||
assert out["X"].geom_type == "Polygon", "sliver should leave X"
|
||||
assert out["N"].contains(x_sliver.representative_point()), "absorbed into N"
|
||||
after = unary_union([out["X"], out["N"]]).area
|
||||
assert after == pytest.approx(before, rel=1e-6), "coverage conserved, not dropped"
|
||||
|
||||
def test_approx_area_matches_real_metric_area(self):
|
||||
# The latitude-scaled area approximation should be within a few percent of
|
||||
# the true projected area for a building-scale polygon.
|
||||
import pyproj
|
||||
from shapely.ops import transform as transform_geometry
|
||||
|
||||
poly = _mbox(0, 0, 50, 60) # ~3000 m²
|
||||
to_bng = pyproj.Transformer.from_crs(
|
||||
"EPSG:4326", "EPSG:27700", always_xy=True
|
||||
)
|
||||
true_m2 = transform_geometry(to_bng.transform, poly).area
|
||||
assert _approx_area_m2(poly) == pytest.approx(true_m2, rel=0.02)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Greenspace subtraction is connectivity-preserving
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGreenspaceReconnect:
|
||||
"""Greenspace that CROSSES a postcode must not shatter it: a narrow strip is
|
||||
bridged back (postcode stays one piece), a wide barrier is left subtracted
|
||||
(postcode genuinely splits)."""
|
||||
|
||||
def test_narrow_strip_reconnects_postcode(self):
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
postcode = box(0, 0, 200, 100)
|
||||
strip = box(95, 0, 105, 100) # 10 m wide green strip crossing the postcode
|
||||
# Sanity: a plain difference WOULD split it into two parts.
|
||||
assert postcode.difference(strip).geom_type == "MultiPolygon"
|
||||
|
||||
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
|
||||
assert result.geom_type == "Polygon", "narrow strip should be bridged"
|
||||
assert result.is_valid
|
||||
|
||||
def test_wide_barrier_keeps_postcode_split(self):
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
postcode = box(0, 0, 200, 100)
|
||||
barrier = box(60, 0, 130, 100) # 70 m wide — beyond 2x the bridge width
|
||||
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
|
||||
assert result.geom_type == "MultiPolygon", "wide barrier should stay split"
|
||||
assert len(result.geoms) == 2
|
||||
|
||||
def test_edge_greenspace_still_trimmed(self):
|
||||
# Greenspace on the edge (not crossing) is trimmed as before; reconnect is
|
||||
# a no-op because the result stays a single piece.
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
postcode = box(0, 0, 100, 100) # 10000 m²
|
||||
park = box(60, 0, 100, 100) # 4000 m² on the right edge
|
||||
result = subtract_greenspace(postcode, STRtree([park]), [park])
|
||||
assert result.geom_type == "Polygon"
|
||||
assert result.area == pytest.approx(6000, rel=0.01)
|
||||
|
||||
def test_result_is_always_polygonal(self):
|
||||
# Regression: _reconnect_split must never return a GeometryCollection
|
||||
# (line/point debris) — downstream to_wgs84_geojson_multi would truncate a
|
||||
# GC to a single part, silently dropping substantial pieces. Sweep many
|
||||
# strip widths/offsets (incl. coincident-edge-prone integer geometries).
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
postcode = box(0, 0, 200, 120)
|
||||
for w in (2, 5, 8, 10, 25, 49, 50, 51, 60, 90):
|
||||
for x0 in (40, 70, 95, 100, 130):
|
||||
strip = box(x0, -10, x0 + w, 130)
|
||||
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
|
||||
assert result.geom_type in ("Polygon", "MultiPolygon"), (
|
||||
f"w={w} x0={x0} produced {result.geom_type}"
|
||||
)
|
||||
assert result.is_valid and not result.is_empty
|
||||
|
||||
def test_wide_barrier_preserves_all_substantial_parts(self):
|
||||
# The motivating case for the GC fix: a wide barrier genuinely splits the
|
||||
# postcode; ALL substantial parts must survive (not be truncated to one).
|
||||
from shapely.strtree import STRtree
|
||||
|
||||
postcode = box(0, 0, 300, 100)
|
||||
barrier = box(120, 0, 200, 100) # 80 m wide -> beyond 2x bridge
|
||||
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
|
||||
assert result.geom_type == "MultiPolygon"
|
||||
areas = sorted(p.area for p in result.geoms)
|
||||
assert areas == pytest.approx([10000, 12000], rel=0.01) # both banks kept
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Augment properties.parquet with estimated current prices.
|
||||
"""Estimate current prices for the merged properties, as a standalone artifact.
|
||||
|
||||
For properties with a known prior sale, applies the repeat-sales price index
|
||||
to adjust the last known price to the current date, then blends with kNN
|
||||
|
|
@ -6,8 +6,13 @@ estimates from nearby recently-sold properties. Includes:
|
|||
- Capping extreme index adjustments
|
||||
- kNN spatial blending
|
||||
|
||||
Modifies properties.parquet in-place. Temporarily joins postcode.parquet
|
||||
for lat/lon needed by kNN, then drops those columns before writing.
|
||||
Reads the slim price_inputs.parquet (built by property_base, independently of
|
||||
merge's area features) plus postcode.parquet for the lat/lon kNN needs, and
|
||||
writes a slim price_estimates.parquet of just the natural key (Postcode +
|
||||
coalesced address) and "Estimated current price" / "Est. price per sqm".
|
||||
join_price_estimates.py joins those two columns back onto properties.parquet.
|
||||
Because the inputs do not depend on merge's area columns, adding such a column
|
||||
does not invalidate this step.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
@ -26,12 +31,27 @@ from pipeline.transform.price_estimation.knn import (
|
|||
from pipeline.transform.price_estimation.utils import (
|
||||
CURRENT_FRAC_YEAR,
|
||||
CURRENT_YEAR,
|
||||
ESTIMATE_COLUMNS,
|
||||
JOIN_KEYS,
|
||||
MAX_LOG_ADJUSTMENT,
|
||||
interpolate_log_index,
|
||||
join_address_expr,
|
||||
sector_expr,
|
||||
type_group_expr,
|
||||
)
|
||||
|
||||
# Columns estimate reads from price_inputs.parquet. The two address columns are
|
||||
# only carried to build the natural join key (Postcode + coalesced address).
|
||||
INPUT_COLUMNS = [
|
||||
"Postcode",
|
||||
"Property type",
|
||||
"Total floor area (sqm)",
|
||||
"Last known price",
|
||||
"Date of last transaction",
|
||||
"Address per Property Register",
|
||||
"Address per EPC",
|
||||
]
|
||||
|
||||
MAX_KNN_TO_INDEX_RATIO = 2.0
|
||||
MIN_KNN_TO_INDEX_RATIO = 0.5
|
||||
# Cap the final estimate at this multiple of the last known price as a guard
|
||||
|
|
@ -161,13 +181,14 @@ def guarded_blend_estimates(
|
|||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Augment properties.parquet with estimated current prices"
|
||||
description="Estimate current prices for the merged properties"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--properties",
|
||||
"--input",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Path to properties.parquet (modified in-place)",
|
||||
help="Path to price_inputs.parquet (slim per-dwelling inputs from "
|
||||
"property_base)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--postcodes",
|
||||
|
|
@ -178,22 +199,23 @@ def main():
|
|||
parser.add_argument(
|
||||
"--index", type=Path, required=True, help="Path to price_index.parquet"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output price_estimates.parquet (natural key + estimate columns)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Loading properties.parquet...")
|
||||
df = pl.read_parquet(args.properties)
|
||||
print(f" {len(df):,} rows, {len(df.columns)} columns")
|
||||
print("Loading price inputs (projection)...")
|
||||
df = pl.read_parquet(args.input, columns=INPUT_COLUMNS)
|
||||
print(f" {len(df):,} rows, {len(INPUT_COLUMNS)} input columns")
|
||||
|
||||
# Join lat/lon from postcode.parquet for kNN spatial queries
|
||||
postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon")
|
||||
df = df.join(postcodes, on="Postcode", how="left")
|
||||
print(f" Joined lat/lon from {len(postcodes):,} postcodes")
|
||||
|
||||
# Drop existing estimated columns if re-running
|
||||
for col in ["Estimated current price", "Est. price per sqm"]:
|
||||
if col in df.columns:
|
||||
df = df.drop(col)
|
||||
|
||||
# Derive helper columns
|
||||
df = df.with_columns(
|
||||
sector_expr().alias("_sector"),
|
||||
|
|
@ -355,16 +377,15 @@ def main():
|
|||
.alias("Est. price per sqm"),
|
||||
)
|
||||
|
||||
# Drop all temporary columns and joined lat/lon (those belong in postcode.parquet)
|
||||
temp_cols = [c for c in df.columns if c.startswith("_") or c.startswith("log_idx_")]
|
||||
df = df.drop(temp_cols).drop("lat", "lon")
|
||||
# Emit only the natural join key and the two estimate columns.
|
||||
# join_price_estimates.py joins these back onto properties.parquet.
|
||||
result = df.with_columns(join_address_expr()).select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
|
||||
|
||||
df.write_parquet(args.properties)
|
||||
size_mb = args.properties.stat().st_size / (1024 * 1024)
|
||||
print(f"\nWrote {args.properties} ({size_mb:.1f} MB)")
|
||||
print(
|
||||
f" {len(df):,} rows, {len(df.columns)} columns (including 'Estimated current price')"
|
||||
)
|
||||
result.write_parquet(args.output)
|
||||
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||
print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
|
||||
n_priced = result.filter(pl.col("Estimated current price").is_not_null()).height
|
||||
print(f" {len(result):,} rows, {n_priced:,} with an estimated current price")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -16,6 +16,26 @@ LATEST_COMPLETE_YEAR = CURRENT_YEAR - 1
|
|||
_today = date.today()
|
||||
CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12
|
||||
|
||||
# The two columns price estimation contributes to properties.parquet, kept here
|
||||
# so both the producer (estimate) and the joiner (join_price_estimates) agree.
|
||||
ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
|
||||
|
||||
# Natural join key from estimates back onto properties: postcode plus the
|
||||
# coalesced register/EPC address. This is unique and non-null on the deduped
|
||||
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
|
||||
# estimates 1:1 onto properties regardless of row order — estimates are computed
|
||||
# from a separate price_inputs.parquet, so a positional key would not line up.
|
||||
JOIN_ADDRESS = "_join_address"
|
||||
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
|
||||
|
||||
|
||||
def join_address_expr() -> pl.Expr:
|
||||
"""The coalesced address half of the natural key, aliased to JOIN_ADDRESS."""
|
||||
return pl.coalesce("Address per Property Register", "Address per EPC").alias(
|
||||
JOIN_ADDRESS
|
||||
)
|
||||
|
||||
|
||||
# Cap on log(index_ratio) to prevent wild estimates from thin sectors
|
||||
MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change
|
||||
TERRACE_TYPES = [
|
||||
|
|
|
|||
217
pipeline/transform/property_base.py
Normal file
217
pipeline/transform/property_base.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
"""Shared property base: the dwelling universe before any area enrichment.
|
||||
|
||||
This is the single source of truth for *which* dwellings exist and their
|
||||
intrinsic, source-level attributes (price, floor area, type, addresses). Both
|
||||
``merge`` (which enriches it with postcode/LSOA-keyed area features to build
|
||||
properties.parquet) and price estimation (which only needs the intrinsic
|
||||
columns) start from exactly these rows, so estimates computed here line up 1:1
|
||||
with the final properties by the natural key ``(Postcode, coalesced address)``.
|
||||
|
||||
Living in its own module is what lets price estimation be *cached* across
|
||||
merge changes: ``price_inputs.parquet`` depends only on epc_pp + arcgis + this
|
||||
file, so adding an area column to merge.py does not invalidate it and the
|
||||
expensive index/kNN steps are skipped.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
||||
|
||||
MIN_FLOOR_AREA_M2 = 10
|
||||
|
||||
# Columns price estimation reads, with the final (properties.parquet) names so
|
||||
# index.py/estimate.py and the join all speak the same schema. The two address
|
||||
# columns form the natural join key (Postcode + their coalesce).
|
||||
PRICE_INPUT_SELECT = [
|
||||
pl.col("postcode").alias("Postcode"),
|
||||
pl.col("total_floor_area").alias("Total floor area (sqm)"),
|
||||
pl.col("latest_price").alias("Last known price"),
|
||||
pl.col("date_of_transfer").alias("Date of last transaction"),
|
||||
"historical_prices",
|
||||
pl.col("pp_address").alias("Address per Property Register"),
|
||||
pl.col("epc_address").alias("Address per EPC"),
|
||||
]
|
||||
|
||||
|
||||
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Return the supported postcode universe with geography join keys."""
|
||||
return (
|
||||
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
|
||||
.filter(pl.col("doterm").is_null())
|
||||
.select(
|
||||
pl.col("pcds").alias("postcode"),
|
||||
"lat",
|
||||
pl.col("long").alias("lon"),
|
||||
"ctry25cd",
|
||||
pl.col("lsoa21cd").alias("lsoa21"),
|
||||
pl.col("oa21cd").alias("oa21"),
|
||||
pl.col("pcon24cd").alias("pcon"),
|
||||
)
|
||||
.drop_nulls(["postcode"])
|
||||
.unique(["postcode"])
|
||||
)
|
||||
|
||||
|
||||
def _remap_terminated_postcodes(
|
||||
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return (
|
||||
wide.join(
|
||||
postcode_mapping,
|
||||
left_on="postcode",
|
||||
right_on="old_postcode",
|
||||
how="left",
|
||||
)
|
||||
.with_columns(
|
||||
pl.coalesce("new_postcode", "postcode").alias("postcode"),
|
||||
)
|
||||
.drop("new_postcode")
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Keep one row per (postcode, address) — the most-recent transaction.
|
||||
|
||||
The terminated-postcode remap can map two distinct postcodes onto one active
|
||||
successor, collapsing the same physical address onto a single
|
||||
(postcode, address) key with conflicting sale records. Keep the row with the
|
||||
latest date_of_transfer so the headline price/date reflect the most recent
|
||||
transaction; genuinely distinct addresses are untouched.
|
||||
|
||||
The dedup key coalesces the price-paid address with the EPC address: EPC-only
|
||||
dwellings (never sold) have a null pp_address, so keying on pp_address alone
|
||||
would collapse EVERY EPC-only dwelling in a postcode onto one
|
||||
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
|
||||
address is unique within its postcode (the EPC frame is deduped on
|
||||
address+postcode upstream), so the coalesced key keeps them distinct while
|
||||
leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
|
||||
a sale exists.
|
||||
"""
|
||||
return (
|
||||
wide.with_columns(
|
||||
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
|
||||
)
|
||||
.sort("date_of_transfer", descending=True, nulls_last=True)
|
||||
.unique(
|
||||
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
|
||||
)
|
||||
.drop("_dedupe_address")
|
||||
)
|
||||
|
||||
|
||||
def _filter_to_active_english_postcodes(
|
||||
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
|
||||
) -> pl.LazyFrame:
|
||||
return wide.join(active_postcodes, on="postcode", how="semi")
|
||||
|
||||
|
||||
def property_type_expr() -> pl.Expr:
|
||||
"""Unaliased property-type expression: prefer EPC, fall back to price-paid.
|
||||
|
||||
For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer
|
||||
detail. Depends only on intrinsic base columns (epc_property_type,
|
||||
pp_property_type, built_form), so merge and price_inputs derive the same
|
||||
value. Callers alias it ("property_type" in merge, "Property type" in
|
||||
price_inputs).
|
||||
"""
|
||||
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
|
||||
["NO DATA!", "Not Recorded"]
|
||||
)
|
||||
has_epc = pl.col("epc_property_type").is_not_null()
|
||||
is_house = pl.col("epc_property_type") == "House"
|
||||
return (
|
||||
pl.when(has_epc & is_house & ~bad_built_form)
|
||||
.then(pl.col("built_form"))
|
||||
.when(has_epc & is_house)
|
||||
.then(pl.col("pp_property_type"))
|
||||
.when(has_epc)
|
||||
.then(pl.col("epc_property_type"))
|
||||
.otherwise(pl.col("pp_property_type"))
|
||||
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
|
||||
# collapse terrace sub-types, and fold rare types into "Other"
|
||||
.replace(
|
||||
{
|
||||
"Flat": "Flats/Maisonettes",
|
||||
"Maisonette": "Flats/Maisonettes",
|
||||
"End-Terrace": "Terraced",
|
||||
"Mid-Terrace": "Terraced",
|
||||
"Enclosed End-Terrace": "Terraced",
|
||||
"Enclosed Mid-Terrace": "Terraced",
|
||||
"Bungalow": "Other",
|
||||
"Park home": "Other",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def build_postcode_centroids(arcgis_path: Path) -> pl.LazyFrame:
|
||||
"""One row per active-English postcode with its lat/lon, from arcgis.
|
||||
|
||||
This is the lat/lon source price estimation needs (index sector centroids,
|
||||
kNN). It is the same per-postcode lat/lon merge writes into postcode.parquet
|
||||
(both come from arcgis), but built straight from arcgis so the index/estimate
|
||||
steps do not depend on the merge output — adding an area column to merge
|
||||
therefore does not invalidate the expensive price index/kNN.
|
||||
"""
|
||||
return _active_english_postcode_area(pl.scan_parquet(arcgis_path)).select(
|
||||
pl.col("postcode").alias("Postcode"), "lat", "lon"
|
||||
)
|
||||
|
||||
|
||||
def build_property_base(epc_pp_path: Path, arcgis_path: Path) -> pl.LazyFrame:
|
||||
"""The deduped, active-English dwelling universe from epc_pp + arcgis.
|
||||
|
||||
Floor filter -> terminated-postcode remap -> collapse-dedupe -> restrict to
|
||||
the active English postcode universe. Returns a LazyFrame with the original
|
||||
epc_pp column names; merge enriches it, the CLI projects it to price_inputs.
|
||||
"""
|
||||
wide = pl.scan_parquet(epc_pp_path).filter(
|
||||
pl.col("total_floor_area").is_null()
|
||||
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
|
||||
)
|
||||
postcode_mapping = build_postcode_mapping(arcgis_path)
|
||||
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
|
||||
wide = _dedupe_collapsed_properties(wide)
|
||||
active_postcodes = (
|
||||
_active_english_postcode_area(pl.scan_parquet(arcgis_path))
|
||||
.select("postcode")
|
||||
.unique()
|
||||
)
|
||||
return _filter_to_active_english_postcodes(wide, active_postcodes)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Write the slim price-estimation inputs from epc_pp + arcgis"
|
||||
)
|
||||
parser.add_argument("--epc-pp", type=Path, required=True)
|
||||
parser.add_argument("--arcgis", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--output", type=Path, required=True, help="price_inputs.parquet output"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--centroids",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="postcode_centroids.parquet output (Postcode, lat, lon)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
base = build_property_base(args.epc_pp, args.arcgis)
|
||||
price_inputs = base.with_columns(
|
||||
property_type_expr().alias("Property type")
|
||||
).select(*PRICE_INPUT_SELECT, "Property type")
|
||||
price_inputs.sink_parquet(args.output)
|
||||
n = pl.scan_parquet(args.output).select(pl.len()).collect().item()
|
||||
print(f"Wrote {args.output} ({args.output.stat().st_size / 1e6:.1f} MB), {n:,} dwellings")
|
||||
|
||||
build_postcode_centroids(args.arcgis).sink_parquet(args.centroids)
|
||||
n_pc = pl.scan_parquet(args.centroids).select(pl.len()).collect().item()
|
||||
print(f"Wrote {args.centroids} ({args.centroids.stat().st_size / 1e6:.1f} MB), {n_pc:,} postcodes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
pipeline/transform/test_area_crime_averages.py
Normal file
81
pipeline/transform/test_area_crime_averages.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import polars as pl
|
||||
|
||||
from pipeline.transform.area_crime_averages import (
|
||||
NATIONAL_AREA,
|
||||
SCOPE_NATIONAL,
|
||||
SCOPE_OUTCODE,
|
||||
SCOPE_SECTOR,
|
||||
compute_area_crime_averages,
|
||||
)
|
||||
|
||||
_BURGLARY = "Burglary (/yr, 7y)"
|
||||
_ROBBERY = "Robbery (/yr, 7y)"
|
||||
|
||||
|
||||
def _postcodes() -> pl.LazyFrame:
|
||||
return pl.LazyFrame(
|
||||
{
|
||||
"Postcode": ["E14 2DG", "E14 2AB", "E14 9XY", "M1 1AE", "E14 2ZZ"],
|
||||
# E14 9XY has no usable crime data; E14 2AB lacks robbery; E14 2ZZ has
|
||||
# crime but (below) no properties, so it must not weight any average.
|
||||
_BURGLARY: [10.0, 20.0, None, 5.0, 100.0],
|
||||
_ROBBERY: [2.0, None, None, 1.0, 50.0],
|
||||
# An unrelated column proves only the crime columns are averaged.
|
||||
"Median age": [40.0, 41.0, 42.0, 30.0, 99.0],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _properties() -> pl.LazyFrame:
|
||||
# Property rows per postcode become the weights (3 / 1 / 2 / 4). E14 2ZZ has
|
||||
# none, so it is excluded entirely.
|
||||
postcodes = ["E14 2DG"] * 3 + ["E14 2AB"] + ["E14 9XY"] * 2 + ["M1 1AE"] * 4
|
||||
return pl.LazyFrame({"Postcode": postcodes})
|
||||
|
||||
|
||||
def _row(df: pl.DataFrame, scope: str, area: str) -> dict:
|
||||
matched = df.filter((pl.col("scope") == scope) & (pl.col("area") == area))
|
||||
assert matched.height == 1, f"expected one {scope} row for {area!r}"
|
||||
return matched.to_dicts()[0]
|
||||
|
||||
|
||||
def test_property_weighted_means_skip_nulls():
|
||||
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||
|
||||
national = _row(result, SCOPE_NATIONAL, NATIONAL_AREA)
|
||||
# Burglary: (10*3 + 20*1 + 5*4) / (3+1+4) = 70/8; E14 9XY null dilutes nothing.
|
||||
assert national[_BURGLARY] == 8.75
|
||||
# Robbery: (2*3 + 1*4) / (3+4) = 10/7; both null postcodes are excluded from
|
||||
# the numerator AND the denominator.
|
||||
assert national[_ROBBERY] == pl.Series([10.0 / 7.0]).cast(pl.Float32).item()
|
||||
|
||||
outcode = _row(result, SCOPE_OUTCODE, "E14")
|
||||
assert outcode[_BURGLARY] == 12.5 # (10*3 + 20*1) / 4
|
||||
assert outcode[_ROBBERY] == 2.0 # only E14 2DG has robbery (2 * 3 / 3)
|
||||
|
||||
|
||||
def test_sector_aggregation_and_all_null_rows_dropped():
|
||||
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||
|
||||
sector = _row(result, SCOPE_SECTOR, "E14 2")
|
||||
assert sector[_BURGLARY] == 12.5
|
||||
assert sector[_ROBBERY] == 2.0
|
||||
|
||||
# E14 9XY has properties but no crime data at all, so its sector "E14 9" is
|
||||
# all-null and must be dropped rather than reported as a known area.
|
||||
assert result.filter(pl.col("area") == "E14 9").height == 0
|
||||
|
||||
|
||||
def test_postcodes_without_properties_are_excluded():
|
||||
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||
|
||||
# E14 2ZZ carries crime values but no properties; including it would pull the
|
||||
# E14 outcode burglary mean toward its 100.0. It must contribute nothing.
|
||||
outcode = _row(result, SCOPE_OUTCODE, "E14")
|
||||
assert outcode[_BURGLARY] == 12.5
|
||||
|
||||
|
||||
def test_only_crime_columns_are_emitted():
|
||||
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||
assert set(result.columns) == {"scope", "area", _BURGLARY, _ROBBERY}
|
||||
assert result.schema[_BURGLARY] == pl.Float32
|
||||
|
|
@ -1,13 +1,10 @@
|
|||
import json
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import pytest
|
||||
import shapely
|
||||
from pyproj import Transformer
|
||||
|
||||
from pipeline.transform.crime_spatial import transform_crime_spatial
|
||||
from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
|
||||
|
||||
_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
|
||||
|
||||
|
|
@ -16,6 +13,10 @@ _CSV_HEADER = (
|
|||
"LSOA code,LSOA name,Crime type,Last outcome category,Context"
|
||||
)
|
||||
|
||||
# Average-annual-count crime column name for a window (the filterable feature).
|
||||
def _raw(t: str, window: str = "7y") -> str:
|
||||
return f"{t} (/yr, {window})"
|
||||
|
||||
|
||||
def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]:
|
||||
lon, lat = _TO_WGS84.transform(x, y)
|
||||
|
|
@ -39,12 +40,12 @@ def _write_boundaries(units_dir, features_by_district: dict[str, list[dict]]) ->
|
|||
(units_dir / f"{district}.geojson").write_text(json.dumps(collection))
|
||||
|
||||
|
||||
def _crime_row(month: str, x, y, crime_type: str) -> str:
|
||||
def _crime_row(month: str, x, y, crime_type: str, location="On or near X", outcome="U") -> str:
|
||||
if x is None or y is None:
|
||||
lon, lat = "", ""
|
||||
else:
|
||||
lon, lat = _bng_to_wgs84(x, y)
|
||||
return f",{month},F,F,{lon},{lat},On or near X,E01000001,L,{crime_type},U,"
|
||||
return f",{month},F,F,{lon},{lat},{location},E01000001,L,{crime_type},{outcome},"
|
||||
|
||||
|
||||
def _write_month(
|
||||
|
|
@ -59,10 +60,22 @@ def _write_month(
|
|||
|
||||
|
||||
def _run(tmp_path, crime, units, **kwargs):
|
||||
output = tmp_path / "crime_by_postcode.parquet"
|
||||
"""Run the transform and return (crime, by_year, records) DataFrames.
|
||||
|
||||
The crime table carries the average-annual-count columns ("{type} (/yr, …)"),
|
||||
i.e. the raw, absolute number of recorded incidents per year.
|
||||
"""
|
||||
crime_out = tmp_path / "crime_by_postcode.parquet"
|
||||
by_year = tmp_path / "crime_by_postcode_by_year.parquet"
|
||||
transform_crime_spatial(crime, units, output, by_year, buffer_m=50.0, **kwargs)
|
||||
return pl.read_parquet(output), pl.read_parquet(by_year)
|
||||
records = tmp_path / "crime_records.parquet"
|
||||
transform_crime_spatial(
|
||||
crime, units, crime_out, by_year, records, buffer_m=50.0, **kwargs
|
||||
)
|
||||
return (
|
||||
pl.read_parquet(crime_out),
|
||||
pl.read_parquet(by_year),
|
||||
pl.read_parquet(records),
|
||||
)
|
||||
|
||||
|
||||
def test_buffer_overlap_counts_for_each_postcode(tmp_path):
|
||||
|
|
@ -95,17 +108,74 @@ def test_buffer_overlap_counts_for_each_postcode(tmp_path):
|
|||
],
|
||||
)
|
||||
|
||||
avg_df, _ = _run(tmp_path, crime, units)
|
||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
||||
# Single covered month -> pooled rate x12.
|
||||
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == 12.0
|
||||
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == 12.0
|
||||
assert rows["AB1 1AA"]["Robbery (avg/yr)"] == 0.0
|
||||
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||
# Single covered month -> pooled raw rate x12.
|
||||
assert rows["AB1 1AA"][_raw("Burglary")] == 12.0
|
||||
assert rows["AB1 1AB"][_raw("Burglary")] == 12.0
|
||||
assert rows["AB1 1AA"][_raw("Robbery")] == 0.0
|
||||
# Only the 49m robbery counts for C; the 51m one and the blank row do not.
|
||||
assert rows["AB1 1AC"]["Robbery (avg/yr)"] == 12.0
|
||||
assert rows["AB1 1AC"]["Burglary (avg/yr)"] == 0.0
|
||||
assert rows["AB1 1AC"][_raw("Robbery")] == 12.0
|
||||
assert rows["AB1 1AC"][_raw("Burglary")] == 0.0
|
||||
# Anti-social behaviour had no coordinate -> nobody gets it.
|
||||
assert all(r["Anti-social behaviour (avg/yr)"] == 0.0 for r in rows.values())
|
||||
assert all(r[_raw("Anti-social behaviour")] == 0.0 for r in rows.values())
|
||||
|
||||
|
||||
def test_counts_are_not_area_normalised(tmp_path):
|
||||
# Three postcodes of very different footprint, each with exactly one incident
|
||||
# in its buffer. The raw count must be 12/yr for ALL of them: area
|
||||
# normalisation has been removed, so footprint no longer changes the number.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units,
|
||||
{
|
||||
"AB1": [
|
||||
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
|
||||
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20
|
||||
_square_feature("AB1 1AC", 5000, 5000, 5040, 5040), # 40x40
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
crime = tmp_path / "crime"
|
||||
_write_month(
|
||||
crime,
|
||||
"2024-01",
|
||||
[
|
||||
_crime_row("2024-01", 1005, 1005, "Burglary"),
|
||||
_crime_row("2024-01", 3005, 3010, "Burglary"),
|
||||
_crime_row("2024-01", 5020, 5020, "Burglary"),
|
||||
],
|
||||
)
|
||||
|
||||
raw_df, _, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||
for pc in ("AB1 1AA", "AB1 1AB", "AB1 1AC"):
|
||||
assert rows[pc][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||
|
||||
|
||||
def test_windows_pool_only_recent_years(tmp_path):
|
||||
# 2-year window vs 7-year window. An incident in the latest year sits in both
|
||||
# windows; one 6 years back sits only in the 7-year window.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
)
|
||||
crime = tmp_path / "crime"
|
||||
# 12 covered months in 2019 (1 burglary), 12 in 2025 (1 burglary). Latest =
|
||||
# 2025: 7y window = 2019..2025 (both), 2y window = 2024..2025 (only 2025).
|
||||
for month in range(1, 13):
|
||||
ym19 = f"2019-{month:02d}"
|
||||
ym25 = f"2025-{month:02d}"
|
||||
_write_month(crime, ym19, [_crime_row(ym19, 1005, 1005, "Burglary")] if month == 1 else [])
|
||||
_write_month(crime, ym25, [_crime_row(ym25, 1005, 1005, "Burglary")] if month == 1 else [])
|
||||
|
||||
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||
row = raw_df.row(0, named=True)
|
||||
# 7y: 2 incidents over 24 covered months -> 1/yr.
|
||||
assert row[_raw("Burglary", "7y")] == pytest.approx(1.0, abs=0.05)
|
||||
# 2y: 1 incident over 12 covered months -> 1/yr (the 2019 one is excluded).
|
||||
assert row[_raw("Burglary", "2y")] == pytest.approx(1.0, abs=0.05)
|
||||
|
||||
|
||||
def test_by_year_annualises_and_rolls_up(tmp_path):
|
||||
|
|
@ -115,7 +185,6 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
|||
)
|
||||
|
||||
crime = tmp_path / "crime"
|
||||
# Point at the centre of AB1 1AA, well inside its buffer.
|
||||
_write_month(
|
||||
crime,
|
||||
"2023-01",
|
||||
|
|
@ -134,7 +203,7 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
|||
],
|
||||
)
|
||||
|
||||
_, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
_, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
assert by_year_df.height == 1
|
||||
cols = set(by_year_df.columns)
|
||||
assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols
|
||||
|
|
@ -150,77 +219,14 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
|||
# 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12).
|
||||
assert serious[2023] == 24.0
|
||||
assert serious[2024] == 12.0
|
||||
# Coverage calendar: both years published, with their month counts.
|
||||
coverage = {c["year"]: c["months"] for c in row["covered_years"]}
|
||||
assert coverage == {2023: 1, 2024: 2}
|
||||
|
||||
|
||||
def test_area_normalisation_divides_out_buffered_catchment(tmp_path):
|
||||
# Three postcodes of increasing footprint, each with exactly one incident in
|
||||
# its buffer. Normalisation rescales by median_catchment / buffered_area, so
|
||||
# the smallest scores highest and the median-sized one is unchanged -- i.e.
|
||||
# the metric is a density. Dividing by the *buffered* catchment (not the raw
|
||||
# polygon) means the fixed buffer-ring floor keeps the spread gentle, so the
|
||||
# tiniest postcode is not blown up out of proportion.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units,
|
||||
{
|
||||
"AB1": [
|
||||
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
|
||||
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20 (median)
|
||||
_square_feature("AB1 1AC", 5000, 5000, 5020, 5020), # 20x20
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
crime = tmp_path / "crime"
|
||||
_write_month(
|
||||
crime,
|
||||
"2024-01",
|
||||
[
|
||||
_crime_row("2024-01", 1005, 1005, "Burglary"),
|
||||
_crime_row("2024-01", 3005, 3010, "Burglary"),
|
||||
_crime_row("2024-01", 5010, 5010, "Burglary"),
|
||||
],
|
||||
)
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
|
||||
# Re-derive the expected values from the same buffered catchment areas: each
|
||||
# postcode is 12/yr before normalisation, then x (median_buf / buffered_area).
|
||||
postcodes, polygons = load_postcode_polygons(units)
|
||||
buf_area = {
|
||||
pc: float(shapely.area(shapely.buffer(poly, 50.0, quad_segs=8)))
|
||||
for pc, poly in zip(postcodes, polygons)
|
||||
}
|
||||
median_buf = float(np.median(list(buf_area.values())))
|
||||
expected = {pc: 12.0 * median_buf / buf_area[pc] for pc in buf_area}
|
||||
|
||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
||||
for pc, exp in expected.items():
|
||||
assert rows[pc]["Burglary (avg/yr)"] == pytest.approx(exp, abs=0.1)
|
||||
|
||||
# Median catchment unchanged; ordering is by inverse buffered area, but the
|
||||
# buffer-ring floor keeps the spread far below the ~4x raw-area ratio.
|
||||
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
||||
small = rows["AB1 1AA"]["Burglary (avg/yr)"]
|
||||
big = rows["AB1 1AC"]["Burglary (avg/yr)"]
|
||||
assert small > 12.0 > big
|
||||
assert small / big < 1.5
|
||||
|
||||
# by-year series carries the same normalisation.
|
||||
small_row = by_year_df.filter(pl.col("postcode") == "AB1 1AA").row(0, named=True)
|
||||
assert small_row["Burglary (by year)"] == [
|
||||
{"year": 2024, "count": pytest.approx(expected["AB1 1AA"], abs=0.1)}
|
||||
]
|
||||
|
||||
|
||||
def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
||||
# Uneven month coverage across years: 2023 has 1 month (2 incidents),
|
||||
# 2024 has 2 months (2 incidents). The headline is the POOLED annualised
|
||||
# rate over all covered months: 4 incidents / 3 months * 12 = 16/yr -- not
|
||||
# the old mean-of-bars (24+12)/2 = 18, which over-weighted thin years.
|
||||
def test_raw_is_pooled_rate_over_covered_months(tmp_path):
|
||||
# Uneven month coverage: 2023 has 1 month (2 incidents), 2024 has 2 months
|
||||
# (2 incidents). The raw figure is the POOLED annualised rate over all covered
|
||||
# months: 4 incidents / 3 months * 12 = 16/yr.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
|
|
@ -238,10 +244,9 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
|||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
||||
_write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")])
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
|
||||
avg = avg_df.row(0, named=True)
|
||||
assert avg["Burglary (avg/yr)"] == pytest.approx(16.0, abs=0.05)
|
||||
assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(16.0, abs=0.05)
|
||||
|
||||
# Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6).
|
||||
row = by_year_df.row(0, named=True)
|
||||
|
|
@ -251,8 +256,7 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
|||
|
||||
def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
||||
# A single robbery in a 24-covered-month window must read as ~0.5/yr (the
|
||||
# long-run pooled rate), NOT 12/yr (the old years-with-incidents mean that
|
||||
# inflated sporadic categories by up to ~15x).
|
||||
# long-run pooled rate), NOT 12/yr.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
|
|
@ -266,14 +270,10 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
|||
rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")]
|
||||
_write_month(crime, f"{year}-{month:02d}", rows)
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||
|
||||
avg = avg_df.row(0, named=True)
|
||||
# 1 incident over 24 covered months -> 0.5/yr.
|
||||
assert avg["Robbery (avg/yr)"] == pytest.approx(0.5, abs=0.05)
|
||||
# The by-year bar still shows the 2023 incident annualised over 12 covered
|
||||
# months (1/yr); 2024 is covered with zero robberies -> no bar, but the
|
||||
# year IS in the coverage list so consumers may render it as a true zero.
|
||||
assert raw_df.row(0, named=True)[_raw("Robbery")] == pytest.approx(0.5, abs=0.05)
|
||||
row = by_year_df.row(0, named=True)
|
||||
bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]}
|
||||
assert bars == {2023: pytest.approx(1.0, abs=0.05)}
|
||||
|
|
@ -283,9 +283,8 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
|||
|
||||
def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
||||
# Two postcodes policed by different forces. force-a publishes 2023+2024;
|
||||
# force-b publishes only 2023 (a 2024 gap, like Greater Manchester). The
|
||||
# b-postcode's headline must pool over force-b's 12 covered months only,
|
||||
# and its by-year series must NOT contain a 2024 bar or coverage entry.
|
||||
# force-b publishes only 2023 (a 2024 gap). The b-postcode's raw figure must
|
||||
# pool over force-b's 12 covered months only.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units,
|
||||
|
|
@ -299,25 +298,21 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
|||
for month in range(1, 13):
|
||||
ym23 = f"2023-{month:02d}"
|
||||
ym24 = f"2024-{month:02d}"
|
||||
# force-a covers AB1 in both years; one burglary per month in 2024.
|
||||
_write_month(crime, ym23, [], force="force-a")
|
||||
_write_month(
|
||||
crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a"
|
||||
)
|
||||
# force-b covers CD1 in 2023 only: one burglary per month.
|
||||
_write_month(
|
||||
crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b"
|
||||
)
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||
|
||||
# force-a postcode: 12 burglaries over 24 covered months -> 6/yr.
|
||||
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
||||
# force-b postcode: 12 burglaries over 12 covered months -> 12/yr. Under
|
||||
# the old global calendar this would have been diluted to 6/yr by the
|
||||
# uncovered 2024.
|
||||
assert rows["CD1 1AA"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
||||
assert rows["AB1 1AA"][_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
|
||||
# force-b postcode: 12 burglaries over 12 covered months -> 12/yr.
|
||||
assert rows["CD1 1AA"][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||
|
||||
by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()}
|
||||
b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]}
|
||||
|
|
@ -328,59 +323,10 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
|||
assert a_coverage == {2023: 12, 2024: 12}
|
||||
|
||||
|
||||
def test_residue_incidents_in_uncovered_years_are_excluded(tmp_path):
|
||||
# force-b stops publishing after 2023, but a force-a file contains a 2024
|
||||
# incident that falls inside the b-postcode's buffer (cross-border residue,
|
||||
# the Greater Manchester pattern). That incident must not produce a 2024
|
||||
# bar for the b-postcode, nor leak into its pooled headline.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units,
|
||||
{
|
||||
"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)],
|
||||
"CD1": [_square_feature("CD1 1AA", 9000, 9000, 9010, 9010)],
|
||||
},
|
||||
)
|
||||
|
||||
crime = tmp_path / "crime"
|
||||
for month in range(1, 13):
|
||||
ym23 = f"2023-{month:02d}"
|
||||
ym24 = f"2024-{month:02d}"
|
||||
_write_month(crime, ym23, [], force="force-a")
|
||||
# b's own 2023 incidents establish force-b as its home force.
|
||||
_write_month(
|
||||
crime,
|
||||
ym23,
|
||||
[_crime_row(ym23, 9005, 9005, "Burglary")] if month <= 6 else [],
|
||||
force="force-b",
|
||||
)
|
||||
# 2024: only force-a publishes; one of its incidents lands in CD1 1AA.
|
||||
_write_month(
|
||||
crime,
|
||||
ym24,
|
||||
[_crime_row(ym24, 9005, 9005, "Burglary")] if month == 1 else [],
|
||||
force="force-a",
|
||||
)
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
||||
|
||||
b_row = avg_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
|
||||
# Pooled over force-b's 12 covered months (2023): 6 incidents -> 6/yr.
|
||||
# The residue 2024 incident is excluded (force-b published 0 months in 2024).
|
||||
assert b_row["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
||||
|
||||
b_by = by_year_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
|
||||
bars = {p["year"]: p["count"] for p in b_by["Burglary (by year)"]}
|
||||
assert set(bars) == {2023}
|
||||
coverage = {c["year"]: c["months"] for c in b_by["covered_years"]}
|
||||
assert coverage == {2023: 12}
|
||||
|
||||
|
||||
def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
|
||||
# 2023 fully covered; 2024 has only 2 published months. With the default
|
||||
# 6-month minimum, 2024 must produce neither a bar (annualising x6 charts
|
||||
# noise) nor a coverage entry -- but its incidents and months still count
|
||||
# toward the pooled headline.
|
||||
# 6-month minimum, 2024 must produce no bar -- but its incidents and months
|
||||
# still count toward the pooled raw figure.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
|
|
@ -394,12 +340,10 @@ def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
|
|||
ym = f"2024-{month:02d}"
|
||||
_write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")])
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||
|
||||
# Pooled: 14 incidents over 14 covered months -> 12/yr.
|
||||
assert avg_df.row(0, named=True)["Burglary (avg/yr)"] == pytest.approx(
|
||||
12.0, abs=0.05
|
||||
)
|
||||
assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||
row = by_year_df.row(0, named=True)
|
||||
bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]}
|
||||
assert set(bars) == {2023}
|
||||
|
|
@ -425,52 +369,119 @@ def test_by_year_output_is_dense_with_coverage(tmp_path):
|
|||
crime = tmp_path / "crime"
|
||||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
assert by_year_df.height == 2
|
||||
|
||||
quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
||||
assert quiet["Burglary (by year)"] is None
|
||||
assert [c["year"] for c in quiet["covered_years"]] == [2024]
|
||||
# And the headline for the quiet postcode is a genuine 0, not null.
|
||||
quiet_avg = avg_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
||||
assert quiet_avg["Burglary (avg/yr)"] == 0.0
|
||||
# The raw figure for the covered, crime-free postcode is a genuine 0, not null.
|
||||
quiet_raw = raw_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
||||
assert quiet_raw[_raw("Burglary")] == 0.0
|
||||
|
||||
|
||||
def test_serious_rollup_avg_yr_equals_sum_of_components(tmp_path):
|
||||
# Burglary only in 2014, Robbery only in 2024 (one incident each, 2 covered
|
||||
# months total). Components pool over the same covered window (each
|
||||
# 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
|
||||
def test_serious_rollup_equals_sum_of_components(tmp_path):
|
||||
# Burglary only in 2023, Robbery only in 2024 (one incident each, 2 covered
|
||||
# months total, both inside the 7-year window). Components pool over the same
|
||||
# covered window (each 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
)
|
||||
|
||||
crime = tmp_path / "crime"
|
||||
_write_month(crime, "2014-01", [_crime_row("2014-01", 1005, 1005, "Burglary")])
|
||||
_write_month(crime, "2023-01", [_crime_row("2023-01", 1005, 1005, "Burglary")])
|
||||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")])
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
|
||||
avg = avg_df.row(0, named=True)
|
||||
assert avg["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
||||
assert avg["Robbery (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
||||
# Rollup == sum of its component (avg/yr) columns.
|
||||
assert avg["Serious crime (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
||||
assert avg["Serious crime (avg/yr)"] == pytest.approx(
|
||||
avg["Burglary (avg/yr)"] + avg["Robbery (avg/yr)"], abs=0.05
|
||||
row = raw_df.row(0, named=True)
|
||||
assert row[_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
|
||||
assert row[_raw("Robbery")] == pytest.approx(6.0, abs=0.05)
|
||||
assert row[_raw("Serious crime")] == pytest.approx(12.0, abs=0.05)
|
||||
assert row[_raw("Serious crime")] == pytest.approx(
|
||||
row[_raw("Burglary")] + row[_raw("Robbery")], abs=0.05
|
||||
)
|
||||
|
||||
# The by-year rollup series remains the per-year sum of the component bars.
|
||||
serious_bars = {
|
||||
p["year"]: p["count"]
|
||||
for p in by_year_df.row(0, named=True)["Serious crime (by year)"]
|
||||
}
|
||||
assert serious_bars == {
|
||||
2014: pytest.approx(12.0, abs=0.05),
|
||||
2023: pytest.approx(12.0, abs=0.05),
|
||||
2024: pytest.approx(12.0, abs=0.05),
|
||||
}
|
||||
|
||||
|
||||
def test_records_capture_each_counted_incident(tmp_path):
|
||||
# Each (incident, postcode) match within the records window becomes a record
|
||||
# row, carrying month/type/location/outcome/coords. A boundary incident
|
||||
# counted for two postcodes appears once per postcode.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units,
|
||||
{
|
||||
"AB1": [
|
||||
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010),
|
||||
_square_feature("AB1 1AB", 1080, 1000, 1090, 1010),
|
||||
]
|
||||
},
|
||||
)
|
||||
crime = tmp_path / "crime"
|
||||
_write_month(
|
||||
crime,
|
||||
"2024-03",
|
||||
[
|
||||
# In the buffer overlap -> recorded for both postcodes.
|
||||
_crime_row("2024-03", 1045, 1005, "Burglary", location="On or near High St", outcome="Under investigation"),
|
||||
# Only in AB1 1AA's buffer; null outcome (police.uk leaves ASB blank).
|
||||
_crime_row("2024-03", 1005, 1005, "Anti-social behaviour", location="On or near Mill Ln", outcome=""),
|
||||
],
|
||||
)
|
||||
|
||||
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
|
||||
assert set(records_df.columns) == {
|
||||
"postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"
|
||||
}
|
||||
# Sorted by postcode.
|
||||
assert records_df["postcode"].is_sorted()
|
||||
# Burglary appears for BOTH postcodes (boundary multiplicity); ASB only for AA.
|
||||
by_pc = records_df.group_by("postcode").agg(pl.col("crime_type").sort())
|
||||
counts = {r["postcode"]: r["crime_type"] for r in by_pc.to_dicts()}
|
||||
assert counts["AB1 1AA"] == ["Anti-social behaviour", "Burglary"]
|
||||
assert counts["AB1 1AB"] == ["Burglary"]
|
||||
# month_index = year*12 + (month-1) for 2024-03.
|
||||
assert set(records_df["month_index"].to_list()) == {2024 * 12 + 2}
|
||||
# Null outcome round-trips as null, not the string "".
|
||||
asb = records_df.filter(pl.col("crime_type") == "Anti-social behaviour").row(0, named=True)
|
||||
assert asb["outcome"] is None
|
||||
assert asb["location"] == "On or near Mill Ln"
|
||||
|
||||
|
||||
def test_records_window_aligns_to_the_headline_calendar_window(tmp_path):
|
||||
# Records must cover exactly the longest (7y) headline window, which is
|
||||
# calendar-year based. With a mid-year latest month (2025-06) the 7y window
|
||||
# is calendar years 2019..2025, so an incident in 2018-09 -- which the
|
||||
# headline excludes -- must also be excluded from the records, even though a
|
||||
# naive rolling 84-month span (ending 2025-06) would wrongly include it. The
|
||||
# first month of the earliest window year (2019-01) is kept.
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||
)
|
||||
crime = tmp_path / "crime"
|
||||
_write_month(crime, "2018-09", [_crime_row("2018-09", 1005, 1005, "Burglary")])
|
||||
_write_month(crime, "2019-01", [_crime_row("2019-01", 1005, 1005, "Burglary")])
|
||||
_write_month(crime, "2025-06", [_crime_row("2025-06", 1005, 1005, "Burglary")])
|
||||
|
||||
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
|
||||
# 2018-09 (year*12+8) is in the rolling 84-month span but NOT the 7y calendar
|
||||
# window, so it is excluded; 2019-01 and 2025-06 are kept.
|
||||
assert set(records_df["month_index"].to_list()) == {2019 * 12 + 0, 2025 * 12 + 5}
|
||||
|
||||
|
||||
def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
|
||||
units = tmp_path / "units"
|
||||
_write_boundaries(
|
||||
|
|
@ -487,11 +498,10 @@ def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
|
|||
],
|
||||
)
|
||||
|
||||
avg_df, _ = _run(tmp_path, crime, units)
|
||||
columns = avg_df.columns
|
||||
# The unknown type is dropped (no column for it) but a warning is emitted.
|
||||
assert "Cyber fraud (avg/yr)" not in columns
|
||||
assert "Burglary (avg/yr)" in columns
|
||||
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||
columns = raw_df.columns
|
||||
assert _raw("Cyber fraud") not in columns
|
||||
assert _raw("Burglary") in columns
|
||||
err = capsys.readouterr().err
|
||||
assert "Cyber fraud" in err
|
||||
assert "WARNING" in err
|
||||
|
|
@ -515,11 +525,11 @@ def test_legacy_crime_types_are_mapped(tmp_path):
|
|||
],
|
||||
)
|
||||
|
||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
row = avg_df.to_dicts()[0]
|
||||
# Single postcode -> area-norm factor 1.0; single covered month -> x12.
|
||||
assert row["Violence and sexual offences (avg/yr)"] == 12.0
|
||||
assert row["Public order (avg/yr)"] == 12.0
|
||||
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||
row = raw_df.to_dicts()[0]
|
||||
# Single covered month (relative to a 2013-latest window) -> x12.
|
||||
assert row[_raw("Violence and sexual offences")] == 12.0
|
||||
assert row[_raw("Public order")] == 12.0
|
||||
|
||||
by_year_row = by_year_df.row(0, named=True)
|
||||
assert by_year_row["Violence and sexual offences (by year)"] == [
|
||||
|
|
|
|||
136
pipeline/transform/test_join_price_estimates.py
Normal file
136
pipeline/transform/test_join_price_estimates.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
"""Tests for joining slim price estimates back onto properties.parquet.
|
||||
|
||||
estimate.py emits (Postcode, coalesced address, estimate columns) and
|
||||
join_estimates attaches them by that natural key. These tests pin the
|
||||
properties that make the key safe: it maps estimates onto the right rows
|
||||
regardless of order (a shuffled estimates frame is the worst case), it is
|
||||
idempotent, and it refuses a partial/foreign estimates file rather than
|
||||
silently nulling prices.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
from pipeline.transform.join_price_estimates import join_estimates
|
||||
from pipeline.transform.price_estimation.utils import (
|
||||
ESTIMATE_COLUMNS,
|
||||
JOIN_ADDRESS,
|
||||
JOIN_KEYS,
|
||||
join_address_expr,
|
||||
)
|
||||
|
||||
N = 200
|
||||
|
||||
|
||||
def _write_merged(path: Path) -> pl.DataFrame:
|
||||
"""properties.parquet with the natural-key columns, a sentinel order column,
|
||||
and no estimates. Half the rows are sale-addressed, half EPC-only, so the
|
||||
coalesce in the key is exercised; every coalesced address is unique."""
|
||||
df = pl.DataFrame(
|
||||
{
|
||||
"Postcode": [f"AA{i % 7} {i % 9}AA" for i in range(N)],
|
||||
"Address per Property Register": [
|
||||
f"reg-{i}" if i % 2 == 0 else None for i in range(N)
|
||||
],
|
||||
"Address per EPC": [f"epc-{i}" if i % 2 == 1 else None for i in range(N)],
|
||||
"order": list(range(N)),
|
||||
"junk": [f"x{i}" for i in range(N)],
|
||||
}
|
||||
)
|
||||
df.write_parquet(path)
|
||||
return df
|
||||
|
||||
|
||||
def _write_estimates(path: Path, merged_path: Path, *, shuffle: bool = True) -> None:
|
||||
"""Estimates keyed by the natural key, derived from the merged file the way
|
||||
estimate.py does. Estimate = order * 1000 so each row is checkable. Shuffled
|
||||
by default to prove order-independence."""
|
||||
est = (
|
||||
pl.read_parquet(merged_path)
|
||||
.with_columns(join_address_expr())
|
||||
.with_columns(
|
||||
(pl.col("order") * 1000).cast(pl.Float64).alias("Estimated current price"),
|
||||
(pl.col("order") * 10).cast(pl.Int32).alias("Est. price per sqm"),
|
||||
)
|
||||
.select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
|
||||
)
|
||||
if shuffle:
|
||||
est = est.sample(fraction=1.0, shuffle=True, seed=7)
|
||||
est.write_parquet(path)
|
||||
|
||||
|
||||
def test_join_attaches_estimates_to_the_right_rows(tmp_path: Path):
|
||||
props = tmp_path / "properties.parquet"
|
||||
estimates = tmp_path / "price_estimates.parquet"
|
||||
_write_merged(props)
|
||||
_write_estimates(estimates, props)
|
||||
|
||||
written = join_estimates(props, estimates)
|
||||
out = pl.read_parquet(props)
|
||||
|
||||
assert written == N
|
||||
assert out.height == N
|
||||
# Order preserved and the address-half of the key is not left behind.
|
||||
assert out["order"].to_list() == list(range(N))
|
||||
assert out["junk"].to_list() == [f"x{i}" for i in range(N)]
|
||||
assert JOIN_ADDRESS not in out.columns
|
||||
# Every row carries its own estimate, matched by key despite the shuffle.
|
||||
assert out["Estimated current price"].to_list() == [float(i * 1000) for i in range(N)]
|
||||
assert out["Est. price per sqm"].to_list() == [i * 10 for i in range(N)]
|
||||
assert out["Estimated current price"].null_count() == 0
|
||||
|
||||
|
||||
def test_rerun_is_idempotent(tmp_path: Path):
|
||||
props = tmp_path / "properties.parquet"
|
||||
estimates = tmp_path / "price_estimates.parquet"
|
||||
_write_merged(props)
|
||||
_write_estimates(estimates, props)
|
||||
|
||||
join_estimates(props, estimates)
|
||||
first = pl.read_parquet(props)
|
||||
join_estimates(props, estimates) # second run on the augmented file
|
||||
second = pl.read_parquet(props)
|
||||
|
||||
assert second.equals(first)
|
||||
assert second.columns.count("Estimated current price") == 1
|
||||
assert second.columns.count("Est. price per sqm") == 1
|
||||
|
||||
|
||||
def test_missing_estimate_is_rejected(tmp_path: Path):
|
||||
"""A property with no matching estimate (diverged dwelling universe) must
|
||||
fail loudly rather than silently leave its price null."""
|
||||
props = tmp_path / "properties.parquet"
|
||||
estimates = tmp_path / "price_estimates.parquet"
|
||||
_write_merged(props)
|
||||
_write_estimates(estimates, props)
|
||||
# Drop one estimate so a property key is no longer covered.
|
||||
pl.read_parquet(estimates).head(N - 1).write_parquet(estimates)
|
||||
|
||||
with pytest.raises(ValueError, match="no matching estimate"):
|
||||
join_estimates(props, estimates)
|
||||
|
||||
|
||||
def test_duplicate_key_is_rejected(tmp_path: Path):
|
||||
props = tmp_path / "properties.parquet"
|
||||
estimates = tmp_path / "price_estimates.parquet"
|
||||
_write_merged(props)
|
||||
_write_estimates(estimates, props)
|
||||
# Force row 1's key to collide with row 0's.
|
||||
est = pl.read_parquet(estimates).sort("Estimated current price")
|
||||
row0 = est.row(0, named=True)
|
||||
est = est.with_columns(
|
||||
pl.when(pl.int_range(pl.len()) == 1)
|
||||
.then(pl.lit(row0["Postcode"]))
|
||||
.otherwise(pl.col("Postcode"))
|
||||
.alias("Postcode"),
|
||||
pl.when(pl.int_range(pl.len()) == 1)
|
||||
.then(pl.lit(row0[JOIN_ADDRESS]))
|
||||
.otherwise(pl.col(JOIN_ADDRESS))
|
||||
.alias(JOIN_ADDRESS),
|
||||
)
|
||||
est.write_parquet(estimates)
|
||||
|
||||
with pytest.raises(ValueError, match="not unique"):
|
||||
join_estimates(props, estimates)
|
||||
|
|
@ -10,14 +10,11 @@ from pipeline.transform.merge import (
|
|||
LISTED_BUILDING_FEATURE,
|
||||
TREE_DENSITY_FEATURE,
|
||||
_LISTING_OVERLAY_SOURCES,
|
||||
_active_english_postcode_area,
|
||||
_build_unmatched_listing_seed_rows,
|
||||
_canonical_postcode_expr,
|
||||
_best_listing_match,
|
||||
_coalesce_direct_epc_columns,
|
||||
_dedupe_collapsed_properties,
|
||||
_fill_property_level_no_defaults,
|
||||
_filter_to_active_english_postcodes,
|
||||
_join_area_side_tables,
|
||||
_finalize_listings,
|
||||
_integrate_listings,
|
||||
|
|
@ -31,7 +28,6 @@ from pipeline.transform.merge import (
|
|||
_matched_listed_building_flags,
|
||||
_postcode_conservation_area_flags,
|
||||
_postcode_listed_building_candidates,
|
||||
_remap_terminated_postcodes,
|
||||
_split_normal_outputs,
|
||||
_tree_density_by_postcode,
|
||||
_validate_lad_source_coverage,
|
||||
|
|
@ -39,6 +35,12 @@ from pipeline.transform.merge import (
|
|||
_validate_postcode_feature_output,
|
||||
_validate_property_postcodes,
|
||||
)
|
||||
from pipeline.transform.property_base import (
|
||||
_active_english_postcode_area,
|
||||
_dedupe_collapsed_properties,
|
||||
_filter_to_active_english_postcodes,
|
||||
_remap_terminated_postcodes,
|
||||
)
|
||||
|
||||
|
||||
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
|
||||
|
|
@ -115,13 +117,16 @@ def test_tree_density_is_area_level_and_survives_the_split() -> None:
|
|||
assert TREE_DENSITY_FEATURE not in properties_df.columns
|
||||
|
||||
|
||||
def test_crime_columns_are_spatial_counts_not_per_capita() -> None:
|
||||
# Crime is now a raw spatial count per postcode; the per-1k-residents
|
||||
# variants were dropped along with the LSOA population denominator.
|
||||
assert "Serious crime (avg/yr)" in _AREA_COLUMNS
|
||||
assert "Minor crime (avg/yr)" in _AREA_COLUMNS
|
||||
assert "Serious crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
|
||||
assert "Minor crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
|
||||
def test_crime_columns_are_average_annual_counts() -> None:
|
||||
# Crime is the average annual recorded incident count (incidents/yr) over
|
||||
# 7-year and 2-year windows; the old per-1,000 "(per 1k/yr, …)" rate columns
|
||||
# are gone.
|
||||
assert "Serious crime (/yr, 7y)" in _AREA_COLUMNS
|
||||
assert "Serious crime (/yr, 2y)" in _AREA_COLUMNS
|
||||
assert "Minor crime (/yr, 7y)" in _AREA_COLUMNS
|
||||
assert "Burglary (/yr, 2y)" in _AREA_COLUMNS
|
||||
assert "Serious crime (avg/yr)" not in _AREA_COLUMNS
|
||||
assert "Minor crime (avg/yr)" not in _AREA_COLUMNS
|
||||
|
||||
|
||||
def test_active_english_postcode_area_filters_to_active_england() -> None:
|
||||
|
|
@ -292,8 +297,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
|
|||
crime = pl.LazyFrame(
|
||||
{
|
||||
"postcode": ["AA1 1AA", "BB2 2BB"],
|
||||
"Serious crime (avg/yr)": [1.0, 2.0],
|
||||
"Minor crime (avg/yr)": [3.0, 4.0],
|
||||
"Serious crime (/yr, 7y)": [1.0, 2.0],
|
||||
"Minor crime (/yr, 7y)": [3.0, 4.0],
|
||||
}
|
||||
)
|
||||
joined = _join_area_side_tables(
|
||||
|
|
@ -343,8 +348,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
|
|||
crime = pl.LazyFrame(
|
||||
{
|
||||
"postcode": ["AB1 2CD", "EF3 4GH"],
|
||||
"Serious crime (avg/yr)": [1.0, 2.0],
|
||||
"Minor crime (avg/yr)": [3.0, 4.0],
|
||||
"Serious crime (/yr, 7y)": [1.0, 2.0],
|
||||
"Minor crime (/yr, 7y)": [3.0, 4.0],
|
||||
}
|
||||
)
|
||||
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
|
||||
|
|
@ -1314,28 +1319,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
|
|||
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
|
||||
|
||||
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
|
||||
# rollup headlines are precomputed values (deliberately NOT the per-type sum,
|
||||
# which would be 10.0 each) so this test proves the merge consumes the
|
||||
# precomputed column rather than re-summing per-type columns.
|
||||
# rollup rate columns are precomputed in crime_spatial and read straight
|
||||
# through unchanged (the merge no longer renames or re-sums them).
|
||||
crime = pl.LazyFrame(
|
||||
{
|
||||
"postcode": ["AA1 1AA"],
|
||||
"Violence and sexual offences (avg/yr)": [1.0],
|
||||
"Robbery (avg/yr)": [2.0],
|
||||
"Burglary (avg/yr)": [3.0],
|
||||
"Possession of weapons (avg/yr)": [4.0],
|
||||
"Anti-social behaviour (avg/yr)": [1.0],
|
||||
"Criminal damage and arson (avg/yr)": [1.0],
|
||||
"Shoplifting (avg/yr)": [1.0],
|
||||
"Bicycle theft (avg/yr)": [1.0],
|
||||
"Theft from the person (avg/yr)": [1.0],
|
||||
"Other theft (avg/yr)": [1.0],
|
||||
"Vehicle crime (avg/yr)": [1.0],
|
||||
"Public order (avg/yr)": [1.0],
|
||||
"Drugs (avg/yr)": [1.0],
|
||||
"Other crime (avg/yr)": [1.0],
|
||||
"Serious crime (avg/yr)": [7.5],
|
||||
"Minor crime (avg/yr)": [4.2],
|
||||
"Burglary (/yr, 7y)": [3.0],
|
||||
"Burglary (/yr, 2y)": [3.5],
|
||||
"Serious crime (/yr, 7y)": [7.5],
|
||||
"Serious crime (/yr, 2y)": [8.0],
|
||||
"Minor crime (/yr, 7y)": [4.2],
|
||||
"Minor crime (/yr, 2y)": [4.6],
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -1364,16 +1358,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
|
|||
by_postcode = {
|
||||
row["postcode"]: row
|
||||
for row in joined.select(
|
||||
"postcode", "serious_crime_avg_yr", "minor_crime_avg_yr"
|
||||
"postcode",
|
||||
"Serious crime (/yr, 7y)",
|
||||
"Minor crime (/yr, 2y)",
|
||||
).iter_rows(named=True)
|
||||
}
|
||||
# Present postcode: rollups are the precomputed headline values, read through
|
||||
# unchanged (NOT the per-type sum of 10.0).
|
||||
assert by_postcode["AA1 1AA"]["serious_crime_avg_yr"] == 7.5
|
||||
assert by_postcode["AA1 1AA"]["minor_crime_avg_yr"] == 4.2
|
||||
# Missing postcode: rollups stay null rather than fabricating 0.0.
|
||||
assert by_postcode["BB2 2BB"]["serious_crime_avg_yr"] is None
|
||||
assert by_postcode["BB2 2BB"]["minor_crime_avg_yr"] is None
|
||||
# Present postcode: rollup rates pass through unchanged.
|
||||
assert by_postcode["AA1 1AA"]["Serious crime (/yr, 7y)"] == 7.5
|
||||
assert by_postcode["AA1 1AA"]["Minor crime (/yr, 2y)"] == 4.6
|
||||
# Missing postcode: rates stay null rather than fabricating 0.0.
|
||||
assert by_postcode["BB2 2BB"]["Serious crime (/yr, 7y)"] is None
|
||||
assert by_postcode["BB2 2BB"]["Minor crime (/yr, 2y)"] is None
|
||||
|
||||
|
||||
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue