Small fixes
This commit is contained in:
parent
54fbcb1ea6
commit
083f8a982e
24 changed files with 1505 additions and 79 deletions
|
|
@ -52,6 +52,13 @@ JUMP_MIN_PRICE = 2_000_000
|
|||
MIN_BUILD_YEAR = 1700
|
||||
MAX_BUILD_YEAR = 2030
|
||||
|
||||
# Open-ended "before YYYY" bands (EPC only emits "before 1900") have no lower
|
||||
# bound. Rather than dropping these mostly Victorian-and-older dwellings, we
|
||||
# publish a representative year this many years under the stated upper bound
|
||||
# ("before 1900" -> 1890) so they still carry a build year and survive an
|
||||
# age-range filter; is_construction_date_approximate flags it as an estimate.
|
||||
OPEN_BAND_YEARS_BEFORE_BOUND = 10
|
||||
|
||||
# Plausibility bounds for raw EPC dimensions. EPC lodgements contain data-entry
|
||||
# errors (0 m storey heights, 116 m "interior height", 9,210 m² floor areas, 99
|
||||
# habitable rooms) that otherwise propagate verbatim into the published per-
|
||||
|
|
@ -68,10 +75,13 @@ def epc_band_to_year(band: pl.Expr) -> pl.Expr:
|
|||
|
||||
EPC age bands are ranges (e.g. ``1950-1966``); we use the band MIDPOINT
|
||||
(1958) rather than the lower bound, which previously biased every band-derived
|
||||
year ~10-15 years too young. Open-ended lower bands (``before 1900``) are too
|
||||
wide to pin to a year and return null. Single-year / ``... onwards`` bands use
|
||||
that year. Already-numeric inputs (a year produced by an earlier call) pass
|
||||
through unchanged. Years outside [MIN_BUILD_YEAR, MAX_BUILD_YEAR] are nulled.
|
||||
year ~10-15 years too young. Open-ended lower bands (``before 1900``) have no
|
||||
lower bound, so we publish a representative year OPEN_BAND_YEARS_BEFORE_BOUND
|
||||
years under the stated upper bound (``before 1900`` -> 1890) rather than
|
||||
dropping the dwelling; is_construction_date_approximate flags it as an
|
||||
estimate. Single-year / ``... onwards`` bands use that year. Already-numeric
|
||||
inputs (a year produced by an earlier call) pass through unchanged. Years
|
||||
outside [MIN_BUILD_YEAR, MAX_BUILD_YEAR] are nulled.
|
||||
"""
|
||||
text = (
|
||||
band.cast(pl.Utf8)
|
||||
|
|
@ -82,7 +92,7 @@ def epc_band_to_year(band: pl.Expr) -> pl.Expr:
|
|||
high = text.str.extract(r"(\d{4})\D+(\d{4})", 2).cast(pl.Int32, strict=False)
|
||||
year = (
|
||||
pl.when(text.str.starts_with("before "))
|
||||
.then(None)
|
||||
.then(low - OPEN_BAND_YEARS_BEFORE_BOUND)
|
||||
.when(high.is_not_null())
|
||||
.then(((low + high) / 2).round(0).cast(pl.Int32))
|
||||
.otherwise(low)
|
||||
|
|
|
|||
|
|
@ -1149,8 +1149,9 @@ def _canonical_epc_property_type_expr() -> pl.Expr:
|
|||
|
||||
def _construction_year_expr(column: str = "construction_age_band") -> pl.Expr:
|
||||
# Use the shared band->midpoint-year mapping so the direct-EPC / listings
|
||||
# path matches join_epc_pp (band midpoint, not lower bound; 'before 1900' and
|
||||
# implausible years -> null). Already-numeric inputs pass through unchanged.
|
||||
# path matches join_epc_pp (band midpoint, not lower bound; 'before 1900' ->
|
||||
# 1890 representative year; implausible years -> null). Already-numeric inputs
|
||||
# pass through unchanged.
|
||||
return epc_band_to_year(pl.col(column))
|
||||
|
||||
|
||||
|
|
@ -1728,7 +1729,7 @@ def _property_match_candidate_frame(wide: pl.LazyFrame) -> pl.DataFrame:
|
|||
|
||||
|
||||
def _index_candidates(
|
||||
candidates: pl.DataFrame, postcode_key: str, uprn_key: str
|
||||
candidates: pl.DataFrame, postcode_key: str, uprn_key: str, address_key: str
|
||||
) -> tuple[dict[str, list[dict]], dict[str, dict]]:
|
||||
"""Index candidate rows for matching, in a single pass over the frame.
|
||||
|
||||
|
|
@ -1736,17 +1737,41 @@ def _index_candidates(
|
|||
fuzzy street-address match; the UPRN index drives the exact match and is
|
||||
postcode-independent, so it still resolves when a listing's postcode is
|
||||
slightly off.
|
||||
|
||||
The EPC register's UPRN is NOT unique: a single building/parent UPRN fans
|
||||
across many distinct flats (up to 58 distinct (address, postcode) rows in
|
||||
the 2026-06 data; ~9k UPRNs collide, touching ~20k epc_pp rows). Such a
|
||||
UPRN cannot serve as a 1:1 exact-match key — it would mis-link a listing to
|
||||
one arbitrary flat — so any UPRN that resolves to more than one distinct
|
||||
``(postcode_key, address_key)`` identity is dropped from ``uprn_index``;
|
||||
those listings fall back to the fuzzy street-address matcher, which
|
||||
disambiguates the specific flat. A UPRN repeated for the SAME identity
|
||||
(one genuine property) is kept.
|
||||
"""
|
||||
buckets: dict[str, list[dict]] = {}
|
||||
uprn_index: dict[str, dict] = {}
|
||||
# uprn -> first_row, plus the identity of that first row; a uprn drops out
|
||||
# the moment a second distinct (postcode, address) identity appears.
|
||||
uprn_first: dict[str, dict] = {}
|
||||
uprn_identity: dict[str, tuple] = {}
|
||||
uprn_dropped: set[str] = set()
|
||||
for row in candidates.iter_rows(named=True):
|
||||
postcode = row.get(postcode_key)
|
||||
if postcode:
|
||||
buckets.setdefault(postcode, []).append(row)
|
||||
uprn = _normalize_uprn(row.get(uprn_key))
|
||||
if uprn and uprn not in uprn_index:
|
||||
uprn_index[uprn] = row
|
||||
return buckets, uprn_index
|
||||
if not uprn or uprn in uprn_dropped:
|
||||
continue
|
||||
identity = (row.get(postcode_key), row.get(address_key))
|
||||
if uprn not in uprn_first:
|
||||
uprn_first[uprn] = row
|
||||
uprn_identity[uprn] = identity
|
||||
elif identity != uprn_identity[uprn]:
|
||||
# Same UPRN, different (postcode, address): a non-unique parent/
|
||||
# building UPRN. Remove it so it cannot act as a 1:1 key.
|
||||
del uprn_first[uprn]
|
||||
del uprn_identity[uprn]
|
||||
uprn_dropped.add(uprn)
|
||||
return buckets, uprn_first
|
||||
|
||||
|
||||
def _best_listing_property_candidate(
|
||||
|
|
@ -1783,7 +1808,10 @@ def _match_listing_properties(
|
|||
return _empty_listing_property_matches()
|
||||
|
||||
buckets, uprn_index = _index_candidates(
|
||||
property_candidates, "_property_match_postcode", "uprn"
|
||||
property_candidates,
|
||||
"_property_match_postcode",
|
||||
"uprn",
|
||||
"_property_match_address",
|
||||
)
|
||||
best_matches = []
|
||||
for listing in listing_matches.iter_rows(named=True):
|
||||
|
|
@ -1909,7 +1937,10 @@ def _match_direct_epc(
|
|||
return _empty_direct_epc_matches()
|
||||
|
||||
buckets, uprn_index = _index_candidates(
|
||||
epc_candidates, "_direct_epc_match_postcode", "_direct_epc_uprn"
|
||||
epc_candidates,
|
||||
"_direct_epc_match_postcode",
|
||||
"_direct_epc_uprn",
|
||||
"_direct_epc_match_address",
|
||||
)
|
||||
street_index, noise_tokens = _index_epc_streets(epc_candidates)
|
||||
street_score_cache: dict[tuple[str, str], list[tuple[int, str]]] = {}
|
||||
|
|
@ -2214,6 +2245,25 @@ class _BuildResult:
|
|||
listings: pl.DataFrame | None = None
|
||||
|
||||
|
||||
# Property-level Yes/No flags default to "No" once all EPC + listings overlays
|
||||
# have been coalesced. A property with no EPC match has no recorded social
|
||||
# tenure / listed status, which is "No", not "unknown": join_epc_pp fills
|
||||
# was_council_house with "No" only for EPC-matched rows (it runs before the
|
||||
# fuzzy join), so without this the ~32% of EPC-unmatched properties would
|
||||
# publish null instead of "No".
|
||||
_PROPERTY_LEVEL_NO_DEFAULT_COLUMNS = (LISTED_BUILDING_FEATURE, "was_council_house")
|
||||
|
||||
|
||||
def _fill_property_level_no_defaults(frame: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Default the property-level Yes/No flag columns to "No" where null."""
|
||||
return frame.with_columns(
|
||||
*(
|
||||
pl.col(column).fill_null("No")
|
||||
for column in _PROPERTY_LEVEL_NO_DEFAULT_COLUMNS
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _build(
|
||||
epc_pp_path: Path,
|
||||
arcgis_path: Path,
|
||||
|
|
@ -2303,7 +2353,12 @@ def _build(
|
|||
)
|
||||
wide = _filter_to_active_english_postcodes(wide, active_postcodes)
|
||||
|
||||
wide = wide.with_columns(pl.col(LISTED_BUILDING_FEATURE).fill_null("No"))
|
||||
# Default property-level Yes/No flags to "No" here: after the listings
|
||||
# overlay coalesce (so a directly-matched "Yes" survives) and before the
|
||||
# rename to "Former council house". This is the single place the FINAL
|
||||
# was_council_house column (null for ~32% EPC-unmatched rows) gets its "No"
|
||||
# default, alongside Listed building.
|
||||
wide = _fill_property_level_no_defaults(wide)
|
||||
|
||||
# NSPL Feb 2026 renamed geographic code columns to {field}{year}cd.
|
||||
# `_active_english_postcode_area` aliases them back to the short canonical
|
||||
|
|
|
|||
|
|
@ -774,7 +774,7 @@ def test_epc_band_to_year_uses_midpoint_and_clamps():
|
|||
"b": [
|
||||
"England and Wales: 1950-1966", # midpoint 1958
|
||||
"1900-1929", # midpoint 1914
|
||||
"England and Wales: before 1900", # too wide -> null
|
||||
"England and Wales: before 1900", # open-ended -> 1900 - 10 = 1890
|
||||
"2012 onwards", # single year
|
||||
"1012", # implausible -> null
|
||||
"2202", # implausible -> null
|
||||
|
|
@ -784,4 +784,4 @@ def test_epc_band_to_year_uses_midpoint_and_clamps():
|
|||
}
|
||||
)
|
||||
years = df.select(epc_band_to_year(pl.col("b")).alias("y"))["y"].to_list()
|
||||
assert years == [1958, 1914, None, 2012, None, None, None, 1958]
|
||||
assert years == [1958, 1914, 1890, 2012, None, None, None, 1958]
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from pipeline.transform.merge import (
|
|||
_best_listing_match,
|
||||
_coalesce_direct_epc_columns,
|
||||
_dedupe_collapsed_properties,
|
||||
_fill_property_level_no_defaults,
|
||||
_filter_to_active_english_postcodes,
|
||||
_join_area_side_tables,
|
||||
_finalize_listings,
|
||||
|
|
@ -943,6 +944,61 @@ def test_match_direct_epc_matches_by_uprn_across_postcodes() -> None:
|
|||
assert matches["_direct_epc_match_method"].to_list() == ["uprn"]
|
||||
|
||||
|
||||
def test_match_direct_epc_ignores_nonunique_building_uprn() -> None:
|
||||
# A parent/building UPRN that resolves to several distinct (address,
|
||||
# postcode) flats cannot act as a 1:1 exact-match key — it would mis-link
|
||||
# the listing to one arbitrary flat. The listing must fall through to the
|
||||
# street-address matcher, which resolves the specific flat.
|
||||
matches = _match_direct_epc(
|
||||
_listing_matches(
|
||||
[
|
||||
{
|
||||
"_listing_uprn": "100000000001",
|
||||
"_listing_match_address": "FLAT 2 EXAMPLE COURT",
|
||||
"_listing_match_postcode": "AA11AA",
|
||||
}
|
||||
]
|
||||
),
|
||||
_direct_epc_candidates(
|
||||
[
|
||||
{
|
||||
"_direct_epc_uprn": "100000000001",
|
||||
"_direct_epc_match_address": "FLAT 1 EXAMPLE COURT",
|
||||
"_direct_epc_address": "Flat 1, Example Court",
|
||||
},
|
||||
{
|
||||
"_direct_epc_uprn": "100000000001",
|
||||
"_direct_epc_match_address": "FLAT 2 EXAMPLE COURT",
|
||||
"_direct_epc_address": "Flat 2, Example Court",
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
assert matches.height == 1
|
||||
# Resolved by address, not the ambiguous building UPRN, and to the right flat.
|
||||
assert matches["_direct_epc_match_method"].to_list() != ["uprn"]
|
||||
assert matches["_direct_epc_address"].to_list() == ["Flat 2, Example Court"]
|
||||
|
||||
|
||||
def test_fill_property_level_no_defaults_defaults_unmatched_to_no() -> None:
|
||||
# EPC-unmatched properties arrive with null was_council_house / Listed
|
||||
# building; they must publish "No" (not null), while a genuine "Yes" is
|
||||
# preserved.
|
||||
frame = pl.LazyFrame(
|
||||
{
|
||||
LISTED_BUILDING_FEATURE: ["Yes", None, "No"],
|
||||
"was_council_house": ["Yes", None, "No"],
|
||||
}
|
||||
)
|
||||
|
||||
out = _fill_property_level_no_defaults(frame).collect()
|
||||
|
||||
assert out["was_council_house"].to_list() == ["Yes", "No", "No"]
|
||||
assert out[LISTED_BUILDING_FEATURE].to_list() == ["Yes", "No", "No"]
|
||||
assert out["was_council_house"].null_count() == 0
|
||||
|
||||
|
||||
def test_match_direct_epc_matches_by_address_in_same_postcode() -> None:
|
||||
matches = _match_direct_epc(
|
||||
_listing_matches([{"_listing_match_address": "1 EXAMPLE ROAD"}]),
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ import json
|
|||
import polars as pl
|
||||
|
||||
from pipeline.transform.transform_poi import (
|
||||
BUS_STOP_DEDUP_RADIUS_M,
|
||||
_load_ofsted_ratings,
|
||||
_school_icon_category_expr,
|
||||
_significant_tokens,
|
||||
osm_groceries_colocated_with_geolytix,
|
||||
osm_stops_near_naptan,
|
||||
transform,
|
||||
transform_grocery_retail_points,
|
||||
)
|
||||
|
|
@ -71,6 +74,61 @@ def test_osm_groceries_colocated_with_geolytix_handles_empty_inputs():
|
|||
assert osm_groceries_colocated_with_geolytix(osm, empty_glx) == []
|
||||
|
||||
|
||||
def test_significant_tokens_keeps_ms_short_brand():
|
||||
# "M&S" strips to "ms" (len 2); without the short-brand allowlist it would
|
||||
# tokenise to set() and never dedupe against the GEOLYTIX "M&S" chain.
|
||||
assert _significant_tokens("M&S") == {"ms"}
|
||||
assert _significant_tokens("M&S Simply Food") == {"ms", "simply", "food"}
|
||||
# Other sub-3-char tokens stay excluded (allowlist must remain minimal).
|
||||
assert _significant_tokens("Co") == set()
|
||||
|
||||
|
||||
def test_osm_groceries_colocated_with_geolytix_dedupes_m_and_s():
|
||||
# GEOLYTIX brand "M&S" must now match a colocated OSM "M&S Simply Food".
|
||||
geolytix = pl.DataFrame({"category": ["M&S"], "lat": [53.0], "lng": [-1.5]})
|
||||
osm = pl.DataFrame(
|
||||
{
|
||||
"id": ["ms-dup"],
|
||||
"name": ["M&S Simply Food"],
|
||||
"lat": [53.00001],
|
||||
"lng": [-1.5],
|
||||
}
|
||||
)
|
||||
assert osm_groceries_colocated_with_geolytix(osm, geolytix, radius_m=50.0) == [
|
||||
"ms-dup"
|
||||
]
|
||||
|
||||
|
||||
def test_osm_groceries_colocated_default_radius_catches_60m_offset():
|
||||
# OSM and GEOLYTIX routinely place the same store 50-100m apart. The widened
|
||||
# 100m default catches a ~60m same-brand duplicate that the old 50m missed.
|
||||
geolytix = pl.DataFrame({"category": ["Tesco"], "lat": [53.0], "lng": [-1.5]})
|
||||
osm = pl.DataFrame(
|
||||
# ~60 m north of the GEOLYTIX point.
|
||||
{"id": ["dup"], "name": ["Tesco Express"], "lat": [53.00054], "lng": [-1.5]}
|
||||
)
|
||||
assert osm_groceries_colocated_with_geolytix(osm, geolytix) == ["dup"]
|
||||
assert osm_groceries_colocated_with_geolytix(osm, geolytix, radius_m=50.0) == []
|
||||
|
||||
|
||||
def test_osm_stops_near_naptan_drops_within_radius_keeps_far():
|
||||
# A ~60m-offset OSM platform duplicates the NaPTAN stop (dropped at the 100m
|
||||
# default); a genuine opposite-direction stop ~160m away is kept.
|
||||
naptan = pl.DataFrame({"id": ["n1"], "lat": [51.5000], "lng": [-0.1000]})
|
||||
osm = pl.DataFrame(
|
||||
{
|
||||
"id": ["dup", "far"],
|
||||
# ~60 m and ~160 m north of the NaPTAN stop.
|
||||
"lat": [51.50054, 51.50144],
|
||||
"lng": [-0.1000, -0.1000],
|
||||
}
|
||||
)
|
||||
assert BUS_STOP_DEDUP_RADIUS_M == 100.0
|
||||
assert osm_stops_near_naptan(osm, naptan) == ["dup"]
|
||||
# At the old 50m radius the duplicate survived (the bug this fix closes).
|
||||
assert osm_stops_near_naptan(osm, naptan, radius_m=50.0) == []
|
||||
|
||||
|
||||
def _write_boundary(tmp_path):
|
||||
"""A FeatureCollection whose single feature covers the London-area test
|
||||
coords used by the transform() fixtures, so in_england_mask keeps them."""
|
||||
|
|
@ -505,6 +563,32 @@ def test_osm_supermarkets_dropped(tmp_path):
|
|||
assert convenience.height == 1
|
||||
|
||||
|
||||
def test_transform_relabels_osm_iceland_to_brand(tmp_path):
|
||||
# OSM tags Iceland as shop/frozen_food -> "Deli & Specialty". A genuine
|
||||
# Iceland store GEOLYTIX lacks (away from any GEOLYTIX point) must survive
|
||||
# AND be relabelled to the "Iceland" brand, not ship mislabelled as a deli.
|
||||
raw = pl.DataFrame(
|
||||
{
|
||||
"id": ["ice1"],
|
||||
"name": ["Iceland"],
|
||||
"category": ["shop/frozen_food"],
|
||||
"lat": [51.40],
|
||||
"lng": [-0.05],
|
||||
}
|
||||
)
|
||||
inputs = _write_transform_inputs(tmp_path, raw)
|
||||
|
||||
out = transform(**inputs).collect()
|
||||
|
||||
row = out.filter(pl.col("name") == "Iceland")
|
||||
assert row.height == 1
|
||||
assert row["group"][0] == "Groceries"
|
||||
assert row["category"][0] == "Iceland"
|
||||
assert row["icon_category"][0] == "Iceland"
|
||||
# No deli pin remains for the relabelled store.
|
||||
assert out.filter(pl.col("category") == "Deli & Specialty").height == 0
|
||||
|
||||
|
||||
def test_transform_grocery_dedup_drops_only_grocery_aspect(tmp_path):
|
||||
# The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at
|
||||
# (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name
|
||||
|
|
|
|||
|
|
@ -1579,8 +1579,11 @@ def transform_gias_schools(
|
|||
# Store". GEOLYTIX is authoritative for its chains, so an OSM grocery row that
|
||||
# sits on top of a GEOLYTIX point AND carries that point's brand name is the
|
||||
# same physical store and is dropped. Independent corner shops never carry a
|
||||
# chain brand, so they are kept.
|
||||
GROCERY_DEDUP_RADIUS_M = 50.0
|
||||
# chain brand, so they are kept. 100m (not 50m) because OSM and GEOLYTIX
|
||||
# routinely place the same branded store 50-100m apart (entrance vs centroid vs
|
||||
# car park); at 50m ~166 same-brand duplicates (Iceland, Morrisons Daily, Tesco
|
||||
# Express, Sainsbury's Local, ...) survived just outside the radius.
|
||||
GROCERY_DEDUP_RADIUS_M = 100.0
|
||||
|
||||
# Brand-token aliases so an OSM name spelt differently from the GEOLYTIX brand
|
||||
# still matches. GEOLYTIX's "Co-op" tokenises to "coop", but OSM frequently
|
||||
|
|
@ -1591,15 +1594,34 @@ _GROCERY_TOKEN_ALIASES = {
|
|||
"cooperatives": "coop",
|
||||
}
|
||||
|
||||
# Brand tokens shorter than the 3-char floor that must still match. "M&S" strips
|
||||
# to "ms" (len 2) and would otherwise tokenise to set(), so M&S never deduped and
|
||||
# OSM M&S grocery rows survived as phantom duplicates of the GEOLYTIX M&S chain.
|
||||
# "M&S" is the only GEOLYTIX brand whose alnum form is <3 chars, so this
|
||||
# allowlist is safe (no other brand collides).
|
||||
_GROCERY_SHORT_BRAND_TOKENS = {"ms"}
|
||||
|
||||
# OSM tags Iceland/Farmfoods stores shop/frozen_food, which CATEGORY_MAP routes
|
||||
# to "Deli & Specialty". The grocery dedup drops the ones colocated with a
|
||||
# GEOLYTIX twin, but genuine stores GEOLYTIX lacks would otherwise survive
|
||||
# mislabelled as deli, shadowing the brand pill and falling outside the headline
|
||||
# grocery metric. Relabel the survivors (matched by name) to the brand so they
|
||||
# line up with the GEOLYTIX rows.
|
||||
OSM_BRAND_RELABEL = {
|
||||
"Iceland": "Iceland",
|
||||
"Farmfoods": "Farmfoods",
|
||||
}
|
||||
|
||||
|
||||
def _significant_tokens(name: str | None) -> set[str]:
|
||||
"""Lower-case alphanumeric tokens of length >= 3 from a POI name (aliased)."""
|
||||
"""Lower-case alphanumeric tokens of length >= 3 from a POI name (aliased),
|
||||
plus short brand tokens in _GROCERY_SHORT_BRAND_TOKENS (e.g. "ms" for M&S)."""
|
||||
if not name:
|
||||
return set()
|
||||
tokens: set[str] = set()
|
||||
for raw in str(name).lower().split():
|
||||
token = "".join(ch for ch in raw if ch.isalnum())
|
||||
if len(token) >= 3:
|
||||
if len(token) >= 3 or token in _GROCERY_SHORT_BRAND_TOKENS:
|
||||
tokens.add(_GROCERY_TOKEN_ALIASES.get(token, token))
|
||||
return tokens
|
||||
|
||||
|
|
@ -1608,7 +1630,14 @@ def _significant_tokens(name: str | None) -> set[str]:
|
|||
# gaps. Where NaPTAN already has a stop within this radius the area is covered,
|
||||
# so the colocated OSM platform is dropped to avoid double-counting; OSM
|
||||
# platforms with no nearby NaPTAN stop (the gaps) are kept.
|
||||
BUS_STOP_DEDUP_RADIUS_M = 50.0
|
||||
#
|
||||
# 100 m, not 50 m: at 50 m ~24.6k OSM platforms 50-100 m from a NaPTAN stop
|
||||
# survived and double-counted (16.7k at 50-75 m, 7.8k at 75-100 m). The NaPTAN
|
||||
# nearest-neighbour spacing (opposite-side-of-road pairs) has a 166.7 m median
|
||||
# (p25=98.7 m), so 100 m clears the 50-100 m duplicate spike while leaving a
|
||||
# genuine opposite-direction stop (~166 m away) in place; 125-150 m would start
|
||||
# dropping those real opposite-side stops.
|
||||
BUS_STOP_DEDUP_RADIUS_M = 100.0
|
||||
|
||||
|
||||
def osm_stops_near_naptan(
|
||||
|
|
@ -1652,9 +1681,9 @@ def osm_groceries_colocated_with_geolytix(
|
|||
|
||||
An OSM Groceries row is a duplicate when a GEOLYTIX point lies within
|
||||
``radius_m`` metres AND that point's brand tokens (its ``category``, e.g.
|
||||
"Tesco", "Co-op") are all present in the OSM row's name — i.e. the same
|
||||
physical branded store. Brands with no token >= 3 chars (e.g. "M&S") never
|
||||
match, so they are conservatively kept rather than risk a false drop.
|
||||
"Tesco", "Co-op", "M&S") are all present in the OSM row's name — i.e. the
|
||||
same physical branded store. Short brands like "M&S" match via
|
||||
_GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept.
|
||||
|
||||
``osm_groceries`` needs columns ``id``, ``name``, ``lat``, ``lng``;
|
||||
``geolytix`` needs ``category`` (the brand), ``lat``, ``lng``.
|
||||
|
|
@ -1774,6 +1803,31 @@ def transform(
|
|||
# pre-deduplicated.
|
||||
lf = lf.unique(subset=["id", "category"], keep="first", maintain_order=True)
|
||||
|
||||
# Relabel OSM Iceland/Farmfoods rows that CATEGORY_MAP put in "Deli &
|
||||
# Specialty" to the brand, so the genuine stores GEOLYTIX lacks (the grocery
|
||||
# dedup drops the colocated twins) line up with the GEOLYTIX brand pills and
|
||||
# the headline grocery metric instead of shadowing them as deli.
|
||||
for _brand_name, _brand in OSM_BRAND_RELABEL.items():
|
||||
_is_brand = (
|
||||
(pl.col("group") == "Groceries")
|
||||
& (pl.col("category") == "Deli & Specialty")
|
||||
& (pl.col("name") == _brand_name)
|
||||
)
|
||||
lf = lf.with_columns(
|
||||
pl.when(_is_brand)
|
||||
.then(pl.lit(_brand))
|
||||
.otherwise(pl.col("category"))
|
||||
.alias("category"),
|
||||
pl.when(_is_brand)
|
||||
.then(pl.lit(_brand))
|
||||
.otherwise(pl.col("icon_category"))
|
||||
.alias("icon_category"),
|
||||
pl.when(_is_brand)
|
||||
.then(pl.lit("\U0001f6d2"))
|
||||
.otherwise(pl.col("emoji"))
|
||||
.alias("emoji"),
|
||||
)
|
||||
|
||||
naptan_df = pl.scan_parquet(naptan_path).collect()
|
||||
mask = in_england_mask(
|
||||
boundary_path,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue