This commit is contained in:
Andras Schmelczer 2026-07-03 18:28:56 +01:00
parent 909e241907
commit 1ee796b282
29 changed files with 250 additions and 126 deletions

View file

@ -14,7 +14,7 @@ def _prop(pid, pin=None):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# _needs_detail_fetch accurate-pin skip # _needs_detail_fetch: accurate-pin skip
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -32,7 +32,7 @@ def test_needs_detail_fetch_disabled_always_fetches(monkeypatch):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# _prime_detail_postcodes worklist selection + concurrent fetch # _prime_detail_postcodes: worklist selection + concurrent fetch
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -97,7 +97,7 @@ def test_prime_is_a_noop_when_disabled_or_cap_zero(monkeypatch):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# _paginate end-to-end (network stubbed): accurate pins fall back to # _paginate end-to-end (network stubbed): accurate pins fall back to
# coordinates, approximate pins use the detail postcode. # coordinates, approximate pins use the detail postcode.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

View file

@ -9,7 +9,7 @@ import zoopla
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# _run_sources Zoopla inline, others in threads, failures isolated # _run_sources: Zoopla inline, others in threads, failures isolated
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -85,13 +85,13 @@ def test_seed_and_save_detail_caches_round_trip(tmp_path):
def test_seed_detail_caches_tolerates_missing_files(tmp_path): def test_seed_detail_caches_tolerates_missing_files(tmp_path):
rightmove._detail_postcode_cache.clear() rightmove._detail_postcode_cache.clear()
# No file written yet seeding must not raise and must leave cache empty. # No file written yet: seeding must not raise and must leave cache empty.
scraper._seed_detail_caches(["rightmove"], tmp_path) scraper._seed_detail_caches(["rightmove"], tmp_path)
assert rightmove._detail_postcode_cache == {} assert rightmove._detail_postcode_cache == {}
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# run_scrape full orchestration wiring (sources stubbed, no network) # run_scrape: full orchestration wiring (sources stubbed, no network)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

View file

@ -1,4 +1,5 @@
from transform import ( from transform import (
build_listing_url,
build_register_address, build_register_address,
clean_listing_address, clean_listing_address,
extract_full_postcode, extract_full_postcode,
@ -173,3 +174,33 @@ def test_rightmove_transform_without_detail_keeps_coordinate_logic() -> None:
assert result is not None assert result is not None
assert result["Postcode"] == "SW9 7AA" assert result["Postcode"] == "SW9 7AA"
assert result["Postcode source"] == "coordinates" assert result["Postcode source"] == "coordinates"
def test_build_listing_url_stamps_channel_from_new_build_flag() -> None:
# Resale gets RES_BUY; new builds get RES_NEW.
assert build_listing_url("/properties/200", False) == (
"https://www.rightmove.co.uk/properties/200#/?channel=RES_BUY"
)
assert build_listing_url("/properties/200", True) == (
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
)
# An existing channel/fragment on the source URL is stripped and re-stamped.
assert build_listing_url("/properties/200#/?channel=RES_BUY", True) == (
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
)
# Missing URL stays empty.
assert build_listing_url("", True) == ""
def test_rightmove_transform_tags_new_builds_res_new() -> None:
# The Rightmove search response marks new-build developments with
# development=True; transform_property must stamp the listing URL RES_NEW.
new_build = {**_rightmove_prop(), "development": True}
result = transform_property(new_build, "SW9", StubPostcodeIndex("SW9 7AA"))
assert result is not None
assert result["Listing URL"].endswith("#/?channel=RES_NEW")
# Ordinary resale (development absent/false) stays RES_BUY.
resale = transform_property(_rightmove_prop(), "SW9", StubPostcodeIndex("SW9 7AA"))
assert resale is not None
assert resale["Listing URL"].endswith("#/?channel=RES_BUY")

View file

@ -103,7 +103,7 @@ def test_parse_detail_geo_merges_location_uprn_with_address_full_address() -> No
def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None: def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None:
# The only `address` twin on the page belongs to a different uprn (a # The only `address` twin on the page belongs to a different uprn (a
# comparable listing). With a uprn to match on, an unrelated twin is never # comparable listing). With a uprn to match on, an unrelated twin is never
# borrowed full_address stays None rather than grabbing the wrong street. # borrowed: full_address stays None rather than grabbing the wrong street.
html = ( html = (
'"location":{"outcode":"NR29",' '"location":{"outcode":"NR29",'
'"coordinates":{"latitude":52.716014,"longitude":1.614495},' '"coordinates":{"latitude":52.716014,"longitude":1.614495},'
@ -185,7 +185,7 @@ def test_parse_detail_geo_returns_none_for_garbage() -> None:
assert parse_detail_geo("<html><body>no data here</body></html>") is None assert parse_detail_geo("<html><body>no data here</body></html>") is None
assert parse_detail_geo("") is None assert parse_detail_geo("") is None
# Coordinates that are not inside a property location/address wrapper (e.g. # Coordinates that are not inside a property location/address wrapper (e.g.
# only an unwrapped POI) yield nothing safe degradation to the outcode. # only an unwrapped POI) yield nothing, safe degradation to the outcode.
assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None

View file

@ -149,7 +149,7 @@ def map_property_type(sub_type: str | None) -> str:
return "Terraced" return "Terraced"
if "house" in lower or "cottage" in lower: if "house" in lower or "cottage" in lower:
return "Detached" return "Detached"
log.warning("Unknown propertySubType: %r mapping to Other", sub_type) log.warning("Unknown propertySubType: %r, mapping to Other", sub_type)
return "Other" return "Other"
@ -267,7 +267,7 @@ def build_register_address(
property's own number or name (e.g. Zoopla detail pages expose property's own number or name (e.g. Zoopla detail pages expose
``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address ``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address
carries the house identifier that the EPC/Price-Paid register addresses also carries the house identifier that the EPC/Price-Paid register addresses also
use turning a fuzzy street match into a near-exact one. Falls back to the use, turning a fuzzy street match into a near-exact one. Falls back to the
plain cleaned address when no number/name is available. plain cleaned address when no number/name is available.
""" """
cleaned = clean_listing_address(raw_address) cleaned = clean_listing_address(raw_address)
@ -282,6 +282,23 @@ def build_register_address(
return f"{number_or_name}, {cleaned}" if cleaned else number_or_name return f"{number_or_name}, {cleaned}" if cleaned else number_or_name
def build_listing_url(property_url: str | None, is_new_build: bool) -> str:
"""Build the canonical Rightmove listing URL with an explicit channel marker.
The search API always echoes ``?channel=RES_BUY`` in ``propertyUrl`` even for
new-build developments (the request channel stays BUY), so the channel is
re-stamped here from the per-listing ``development`` flag: ``RES_NEW`` for new
builds, ``RES_BUY`` for ordinary resale. The map UI reads this marker to split
new vs non-new listings. Any channel/fragment already on ``propertyUrl`` is
stripped first so the result is deterministic.
"""
if not property_url:
return ""
base = property_url.split("#", 1)[0].split("?", 1)[0]
channel = "RES_NEW" if is_new_build else "RES_BUY"
return f"{RIGHTMOVE_BASE}{base}#/?channel={channel}"
def transform_property( def transform_property(
prop: dict, prop: dict,
outcode: str, outcode: str,
@ -337,7 +354,7 @@ def transform_property(
inferred_postcode = pc_index.nearest(lat, lng) inferred_postcode = pc_index.nearest(lat, lng)
if not inferred_postcode: if not inferred_postcode:
log.debug("No England postcode for property at %.4f, %.4f skipping", lat, lng) log.debug("No England postcode for property at %.4f, %.4f; skipping", lat, lng)
return None return None
raw_address = prop.get("displayAddress", "") or "" raw_address = prop.get("displayAddress", "") or ""
extracted_postcode = extract_full_postcode(raw_address) extracted_postcode = extract_full_postcode(raw_address)
@ -391,7 +408,7 @@ def transform_property(
"price_frequency": "", "price_frequency": "",
"Price qualifier": price_qualifier, "Price qualifier": price_qualifier,
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")), "Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
"Listing URL": RIGHTMOVE_BASE + property_url if property_url else "", "Listing URL": build_listing_url(property_url, bool(prop.get("development"))),
"Listing features": key_features, "Listing features": key_features,
"first_visible_date": prop.get("firstVisibleDate", ""), "first_visible_date": prop.get("firstVisibleDate", ""),
} }

View file

@ -1,7 +1,7 @@
"""Zoopla scraping via FlareSolverr (no browser/VNC needed). """Zoopla scraping via FlareSolverr (no browser/VNC needed).
FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which
still contains the React Server Components flight stream so the existing pure still contains the React Server Components flight stream, so the existing pure
parsers work unchanged: parsers work unchanged:
- the search page yields the outcode's listing detail URLs, and - the search page yields the outcode's listing detail URLs, and
- each detail page's flight stream carries the property's location object - each detail page's flight stream carries the property's location object

View file

@ -12,6 +12,7 @@ import type { TFunction } from 'i18next';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { cellToLatLng, polygonToCells } from 'h3-js'; import { cellToLatLng, polygonToCells } from 'h3-js';
import PriceHistoryChart from '../map/PriceHistoryChart'; import PriceHistoryChart from '../map/PriceHistoryChart';
import { MapErrorBoundary } from '../map/MapErrorBoundary';
import StackedBarChart from '../map/StackedBarChart'; import StackedBarChart from '../map/StackedBarChart';
import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions'; import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions';
import { DualHistogram } from '../map/DualHistogram'; import { DualHistogram } from '../map/DualHistogram';
@ -214,10 +215,13 @@ const SHOWCASE_MAP_START_VIEW: ViewState = {
bearing: 0, bearing: 0,
}; };
// End the Match fly-in over Greater London so the "match" step lands on the same
// place the Inspect (SW5 9AA) and Scout (SW5/SE22/N4) steps then drill into. The
// single-search narrative used to break by zooming to Birmingham then showing London.
const SHOWCASE_MAP_END_VIEW: ViewState = { const SHOWCASE_MAP_END_VIEW: ViewState = {
longitude: -1.89, longitude: -0.12,
latitude: 52.49, latitude: 51.51,
zoom: 8.72, zoom: 9.05,
pitch: 0, pitch: 0,
bearing: 0, bearing: 0,
}; };
@ -260,7 +264,12 @@ function buildShowcaseMapData(): HexagonData[] {
} }
const SHOWCASE_MAP_DATA = buildShowcaseMapData(); const SHOWCASE_MAP_DATA = buildShowcaseMapData();
const SHOWCASE_MAP_TOTAL_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => sum + item.count, 0); // Count only the hexes around Greater London so the Match card's figure matches the
// area it has zoomed to, instead of mislabelling the England-wide total as one city.
const SHOWCASE_LONDON_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => {
const nearLondon = Math.abs(item.lat - 51.5072) < 0.55 && Math.abs(item.lon + 0.1276) < 0.85;
return nearLondon ? sum + item.count : sum;
}, 0);
const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = []; const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = [];
const EMPTY_SHOWCASE_POIS: POI[] = []; const EMPTY_SHOWCASE_POIS: POI[] = [];
@ -601,37 +610,42 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) {
return ( return (
<div className="pointer-events-none relative h-full overflow-hidden bg-warm-100 dark:bg-navy-950/50"> <div className="pointer-events-none relative h-full overflow-hidden bg-warm-100 dark:bg-navy-950/50">
{shouldRenderMap && ( {shouldRenderMap && (
<Suspense fallback={<div className="h-full bg-navy-950/40" aria-hidden="true" />}> // The full deck.gl/maplibre map sits above the fold; contain any WebGL
<ProductMap // context-loss crash to this pane (and auto-recover) instead of letting it
data={SHOWCASE_MAP_DATA} // bubble to the top-level boundary and blank the hero.
postcodeData={EMPTY_SHOWCASE_POSTCODES} <MapErrorBoundary>
usePostcodeView={false} <Suspense fallback={<div className="h-full bg-navy-950/40" aria-hidden="true" />}>
pois={EMPTY_SHOWCASE_POIS} <ProductMap
onViewChange={noopViewChange} data={SHOWCASE_MAP_DATA}
viewFeature={null} postcodeData={EMPTY_SHOWCASE_POSTCODES}
colorRange={null} usePostcodeView={false}
filterRange={null} pois={EMPTY_SHOWCASE_POIS}
viewSource={null} onViewChange={noopViewChange}
onCancelPin={() => {}} viewFeature={null}
features={DEMO_FEATURES} colorRange={null}
selectedHexagonId={null} filterRange={null}
hoveredHexagonId={null} viewSource={null}
onHexagonClick={noopHexagonClick} onCancelPin={() => {}}
onHexagonHover={noopHexagonHover} features={DEMO_FEATURES}
initialViewState={viewState} selectedHexagonId={null}
theme="dark" hoveredHexagonId={null}
screenshotMode onHexagonClick={noopHexagonClick}
hideLegend onHexagonHover={noopHexagonHover}
densityLabel={t('home.showcaseMatchingHomesLabel')} initialViewState={viewState}
totalCount={SHOWCASE_MAP_TOTAL_COUNT} theme="dark"
/> screenshotMode
</Suspense> hideLegend
densityLabel={t('home.showcaseMatchingHomesLabel')}
totalCount={SHOWCASE_LONDON_COUNT}
/>
</Suspense>
</MapErrorBoundary>
)} )}
<div className="pointer-events-none absolute bottom-4 left-4 max-w-[16rem] rounded-md border border-white/15 bg-navy-950/95 px-4 py-3 text-white shadow-2xl shadow-navy-950/35"> <div className="pointer-events-none absolute bottom-4 left-4 max-w-[16rem] rounded-md border border-white/15 bg-navy-950/95 px-4 py-3 text-white shadow-2xl shadow-navy-950/35">
<div className="text-sm font-black leading-none">Birmingham</div> <div className="text-sm font-black leading-none">{t('home.showcaseStep2Region')}</div>
<div className="mt-1 text-xs font-bold leading-tight text-warm-200"> <div className="mt-1 text-xs font-bold leading-tight text-warm-200">
{t('home.showcaseMatchingHomes', { {t('home.showcaseMatchingHomes', {
value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(), value: SHOWCASE_LONDON_COUNT.toLocaleString(),
})} })}
</div> </div>
<div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400"> <div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400">
@ -789,19 +803,22 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
postcode: 'SW5 9AA', postcode: 'SW5 9AA',
score: '94%', score: '94%',
commute: t('home.showcaseMinutes', { count: 23 }), commute: t('home.showcaseMinutes', { count: 23 }),
price: '£492k', perSqm: '£8,200',
delta: '16%',
}, },
{ {
postcode: 'SE22 8EF', postcode: 'SE22 8EF',
score: '91%', score: '91%',
commute: t('home.showcaseMinutes', { count: 28 }), commute: t('home.showcaseMinutes', { count: 28 }),
price: '£518k', perSqm: '£5,900',
delta: '24%',
}, },
{ {
postcode: 'N4 2AB', postcode: 'N4 2AB',
score: '88%', score: '88%',
commute: t('home.showcaseMinutes', { count: 31 }), commute: t('home.showcaseMinutes', { count: 31 }),
price: '£476k', perSqm: '£6,400',
delta: '21%',
}, },
]; ];
@ -934,7 +951,12 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
<div className="truncate border-r border-warm-200 px-2 py-1.5 dark:border-navy-700 sm:px-3 sm:py-2.5"> <div className="truncate border-r border-warm-200 px-2 py-1.5 dark:border-navy-700 sm:px-3 sm:py-2.5">
{row.commute} {row.commute}
</div> </div>
<div className="truncate px-2 py-1.5 font-black sm:px-3 sm:py-2.5">{row.price}</div> <div className="px-2 py-1.5 sm:px-3 sm:py-2.5">
<div className="truncate font-black tabular-nums">{row.perSqm}</div>
<div className="text-[10px] font-bold tabular-nums text-emerald-600 dark:text-emerald-300">
{row.delta}
</div>
</div>
</div> </div>
))} ))}
</div> </div>
@ -1031,6 +1053,7 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
const [isStagePaused, setIsStagePaused] = useState(false); const [isStagePaused, setIsStagePaused] = useState(false);
const [hasStarted, setHasStarted] = useState(false); const [hasStarted, setHasStarted] = useState(false);
const [canPauseOnHover, setCanPauseOnHover] = useState(false); const [canPauseOnHover, setCanPauseOnHover] = useState(false);
const [prefersReducedMotion, setPrefersReducedMotion] = useState(false);
const showcaseRef = useRef<HTMLDivElement | null>(null); const showcaseRef = useRef<HTMLDivElement | null>(null);
const inspectUserScrolledRef = useRef(false); const inspectUserScrolledRef = useRef(false);
@ -1101,6 +1124,27 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
return () => mediaQuery.removeEventListener('change', updateCanPause); return () => mediaQuery.removeEventListener('change', updateCanPause);
}, []); }, []);
useEffect(() => {
if (typeof window.matchMedia !== 'function') return;
const mediaQuery = window.matchMedia('(prefers-reduced-motion: reduce)');
const update = () => setPrefersReducedMotion(mediaQuery.matches);
update();
mediaQuery.addEventListener('change', update);
return () => mediaQuery.removeEventListener('change', update);
}, []);
// Under prefers-reduced-motion the progress-bar animation is disabled in CSS, so
// its onAnimationEnd (which normally advances the carousel) never fires and the
// demo would freeze on step 1. Drive the advance from a timer in that case.
useEffect(() => {
if (!prefersReducedMotion || !isProgressRunning) return;
const timer = window.setTimeout(
() => setActiveStep((step) => (step + 1) % SHOWCASE_STEP_COUNT),
activeStepIntervalMs
);
return () => window.clearTimeout(timer);
}, [prefersReducedMotion, isProgressRunning, activeStep, activeStepIntervalMs]);
const pauseForHover = () => { const pauseForHover = () => {
if (canPauseOnHover) setIsStagePaused(true); if (canPauseOnHover) setIsStagePaused(true);
}; };

View file

@ -235,6 +235,7 @@ export interface HistoricalPrice {
year: number; year: number;
month: number; month: number;
price: number; price: number;
is_new: boolean;
} }
export interface Property { export interface Property {
@ -386,7 +387,7 @@ export interface HexagonStatsResponse {
* sector, shown alongside the national average for each crime metric. */ * sector, shown alongside the national average for each crime metric. */
crime_area_averages?: CrimeAreaAverage[]; crime_area_averages?: CrimeAreaAverage[];
/** Total individual crime records (last 7 years) across the selection's /** Total individual crime records (last 7 years) across the selection's
* postcodes the count behind the "individual crimes" list. */ * postcodes, the count behind the "individual crimes" list. */
crime_total_records?: number; crime_total_records?: number;
central_postcode?: string; central_postcode?: string;
/** Total usual residents (ONS Census 2021) across the postcodes in this /** Total usual residents (ONS Census 2021) across the postcodes in this

View file

@ -3,7 +3,7 @@
Downloads five-year age band counts (TS007A) from the NOMIS API, then computes Downloads five-year age band counts (TS007A) from the NOMIS API, then computes
the median age per LSOA using linear interpolation within the median class. the median age per LSOA using linear interpolation within the median class.
Source: NOMIS (ONS Census 2021 TS007A dataset, NM_2020_1) Source: NOMIS (ONS Census 2021, TS007A dataset, NM_2020_1)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
""" """
@ -20,7 +20,7 @@ from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv
BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE" BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE"
# Five-year age bands in order, with lower bounds for interpolation. # Five-year age bands in order, with lower bounds for interpolation.
# The last band (85+) is open-ended — we treat it as 85-89 for median purposes. # The last band (85+) is open-ended. We treat it as 85-89 for median purposes.
AGE_BANDS = [ AGE_BANDS = [
(0, 5), # Aged 0 to 4 years (0, 5), # Aged 0 to 4 years
(5, 5), # Aged 5 to 9 years (5, 5), # Aged 5 to 9 years

View file

@ -28,7 +28,7 @@ LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU"
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g. # The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a # "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family # TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
# to its own "DLR station" category restoring DLR to the train/tube station # to its own "DLR station" category, restoring DLR to the train/tube station
# list and giving it the DLR roundel rather than a generic tram icon. # list and giving it the DLR roundel rather than a generic tram icon.
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL" LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
@ -337,7 +337,7 @@ class StationAccumulator:
# A single node is never both (ZZLU vs ZZDL), but a co-located # A single node is never both (ZZLU vs ZZDL), but a co-located
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU # interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
# and DLR halves into one group carrying both flags; Tube is checked # and DLR halves into one group carrying both flags; Tube is checked
# first so these resolve to "Tube station" — their primary identity — # first so these resolve to "Tube station" (their primary identity),
# leaving "DLR station" for the DLR-only stops the fix targets. # leaving "DLR station" for the DLR-only stops the fix targets.
if self.category == TRAM_METRO_CATEGORY and self.is_lu: if self.category == TRAM_METRO_CATEGORY and self.is_lu:
return TUBE_STATION_CATEGORY return TUBE_STATION_CATEGORY

View file

@ -85,24 +85,24 @@ RESOLUTION = NATIVE_RESOLUTION
# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is # Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
# otherwise inside the raster means "modelled below the lowest reporting band", # otherwise inside the raster means "modelled below the lowest reporting band",
# i.e. genuinely quiet — see noise_overlay_tiles.py:167. # i.e. genuinely quiet (see noise_overlay_tiles.py:167).
NOISE_NODATA_SENTINEL = np.float32(-96.0) NOISE_NODATA_SENTINEL = np.float32(-96.0)
# Lowest modelled Defra Lden reporting band (dB). Verified against the actual # Lowest modelled Defra Lden reporting band (dB). Verified against the actual
# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in # rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor # (0, 40). Below the band, cells are encoded as 0.0 (genuinely quiet). We floor
# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet" # in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the # (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45. # ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles. # NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
# NOISE_COLOR_STOPS[0]) a rendering threshold, not the data's reporting floor. # NOISE_COLOR_STOPS[0]), a rendering threshold, not the data's reporting floor.
NOISE_QUIET_FLOOR_DB = np.float32(40.0) NOISE_QUIET_FLOOR_DB = np.float32(40.0)
# Sample noise at the postcode representative point itself (no neighbourhood # Sample noise at the postcode representative point itself (no neighbourhood
# window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha # window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha
# of every postcode; because Defra road contours hug every modelled road and # of every postcode; because Defra road contours hug every modelled road and
# representative points sit on/near streets, that inflated postcode noise by # representative points sit on/near streets, that inflated postcode noise by
# roughly +9 dB (log scale) making ~94% of England read >=55 dB Lden and # roughly +9 dB (log scale), making ~94% of England read >=55 dB Lden and
# collapsing the metric's discrimination at the quiet end. Radius 0 -> # collapsing the metric's discrimination at the quiet end. Radius 0 ->
# filter_size 1 -> the maximum_filter is skipped and each postcode reads the # filter_size 1 -> the maximum_filter is skipped and each postcode reads the
# 10m cell it actually sits in. # 10m cell it actually sits in.
@ -110,8 +110,8 @@ POSTCODE_NOISE_RADIUS_M = 0
# Adjacent download tiles overlap by the sampling radius so every postcode's # Adjacent download tiles overlap by the sampling radius so every postcode's
# sampling footprint is fully contained in at least one tile. With point # sampling footprint is fully contained in at least one tile. With point
# sampling (radius 0) this is 0 a representative point falls inside exactly # sampling (radius 0) this is 0 (a representative point falls inside exactly
# one tile but the relationship is kept so any future non-zero radius keeps # one tile), but the relationship is kept so any future non-zero radius keeps
# its window seam-safe. # its window seam-safe.
TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
@ -273,7 +273,7 @@ def _download_tile(
NoGeoTiffError, NoGeoTiffError,
httpx.HTTPStatusError, httpx.HTTPStatusError,
# TransportError is the superset of TimeoutException, ConnectError, # TransportError is the superset of TimeoutException, ConnectError,
# ReadError and ProtocolError including RemoteProtocolError, raised # ReadError and ProtocolError, including RemoteProtocolError, raised
# when the WCS server closes the connection mid-stream ("incomplete # when the WCS server closes the connection mid-stream ("incomplete
# chunked read"). All are transient; retry/split rather than letting # chunked read"). All are transient; retry/split rather than letting
# one flaky tile crash the whole raster download. # one flaky tile crash the whole raster download.
@ -446,7 +446,7 @@ def sample_noise_at_postcodes(
# Defra rasters encode TRUE nodata as the -96.0 sentinel (and # Defra rasters encode TRUE nodata as the -96.0 sentinel (and
# occasionally non-finite / dataset.nodata); genuinely quiet ground # occasionally non-finite / dataset.nodata); genuinely quiet ground
# below the model's lowest reporting band is encoded as 0.0. Only # below the model's lowest reporting band is encoded as 0.0. Only
# the former is "we don't know" — the latter is a real "we know it's # the former is "we don't know". The latter is a real "we know it's
# quiet" reading and must not collapse to null. So treat ONLY true # quiet" reading and must not collapse to null. So treat ONLY true
# nodata as -inf (it never wins a max and never counts as coverage), # nodata as -inf (it never wins a max and never counts as coverage),
# and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a # and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
@ -555,7 +555,7 @@ def main() -> None:
if not tile_paths: if not tile_paths:
print( print(
f"[{label}] WARNING: No tiles downloaded column will be all null" f"[{label}] WARNING: No tiles downloaded; column will be all null"
) )
series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32) series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32)
else: else:

View file

@ -7,7 +7,7 @@ site id and the site's polygon centroid. Sites without access points fall
back to polygon centroids. back to polygon centroids.
Using access points rather than polygon centroids gives much more accurate Using access points rather than polygon centroids gives much more accurate
distance calculations a property next to Hyde Park won't show 400m just distance calculations: a property next to Hyde Park won't show 400m just
because the centroid is in the middle of the park. The site id / centroid because the centroid is in the middle of the park. The site id / centroid
columns let downstream consumers (poi_proximity) collapse the frame back to columns let downstream consumers (poi_proximity) collapse the frame back to
one row per SITE for counting, so a park with 30 gates counts as one park. one row per SITE for counting, so a park with 30 gates counts as one park.
@ -190,7 +190,7 @@ def download_greenspace(output: Path) -> None:
print(f"Reading {site_shps[0].name} for function types...") print(f"Reading {site_shps[0].name} for function types...")
site_funcs = _read_site_functions(site_shps[0]) site_funcs = _read_site_functions(site_shps[0])
# Step 2: Read access points (primary park entrances) # Step 2: Read access points (primary: park entrances)
print(f"Reading {access_shps[0].name}...") print(f"Reading {access_shps[0].name}...")
ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points( ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points(
access_shps[0], site_funcs access_shps[0], site_funcs

View file

@ -930,7 +930,7 @@ def main() -> None:
df.write_parquet(args.output) df.write_parquet(args.output)
print(f"Saved to {args.output}") print(f"Saved to {args.output}")
else: else:
print("No places found skipping output") print("No places found, skipping output")
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -35,7 +35,7 @@ def _data_rows(df: pl.DataFrame) -> pl.DataFrame:
The preamble length varies (title, optional "This worksheet contains..." The preamble length varies (title, optional "This worksheet contains..."
note, then the header row starting with "Time period"), so locate the note, then the header row starting with "Time period"), so locate the
header by content instead of counting rows a fixed slice leaves the header by content instead of counting rows: a fixed slice leaves the
header in the data whenever ONS adds or removes a note line. header in the data whenever ONS adds or removes a note line.
""" """
header_marker = ( header_marker = (
@ -78,7 +78,7 @@ def _latest_rents_long(df: pl.DataFrame) -> pl.DataFrame:
print(f"LAs in latest month: {df.height}") print(f"LAs in latest month: {df.height}")
# Melt to long format: one row per area x bedroom count. # Melt to long format: one row per area x bedroom count.
# PIPR has no Studio category one-bed rent used as proxy for bedrooms=0. # PIPR has no Studio category: one-bed rent used as proxy for bedrooms=0.
frames = [] frames = []
for col, bedrooms in [ for col, bedrooms in [
("rent_1bed", 0), # Studio (proxy) ("rent_1bed", 0), # Studio (proxy)

View file

@ -2,8 +2,8 @@
Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9) Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9)
from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf
categories into our 3 output buckets Owner occupied / Social rent / categories into our 3 output buckets (Owner occupied / Social rent /
Private rent emitting one row per LSOA with the percentage of households in Private rent), emitting one row per LSOA with the percentage of households in
each. The three buckets sum to 100%, so downstream they render as a each. The three buckets sum to 100%, so downstream they render as a
composition/ratio (like the ethnicity and qualifications stacked bars) AND each composition/ratio (like the ethnicity and qualifications stacked bars) AND each
percentage is independently filterable. percentage is independently filterable.
@ -19,7 +19,7 @@ NOTE this table counts HOUSEHOLDS (not usual residents). The join key downstream
(merge.py) is `lsoa21`, the same key used for ethnicity, qualifications, (merge.py) is `lsoa21`, the same key used for ethnicity, qualifications,
median age, and IoD. median age, and IoD.
Source: NOMIS (ONS Census 2021 TS054 dataset, NM_2072_1) Source: NOMIS (ONS Census 2021, TS054 dataset, NM_2072_1)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
""" """

View file

@ -72,7 +72,7 @@ def test_ethnicity_routes_chinese_to_east_and_other_asian_to_se():
def test_ethnicity_percentages_independent_per_lsoa(): def test_ethnicity_percentages_independent_per_lsoa():
"""Two LSOAs get independent profiles the LSOA granularity is the point.""" """Two LSOAs get independent profiles: the LSOA granularity is the point."""
df = pl.concat( df = pl.concat(
[ [
pl.DataFrame( pl.DataFrame(

View file

@ -281,7 +281,7 @@ def test_sample_noise_preserves_genuine_reading_above_quiet_floor(
# The lowest Defra reporting band is 40.0 dB; genuine readings populate # The lowest Defra reporting band is 40.0 dB; genuine readings populate
# [40, ~80]. A genuine in-coverage reading at or just above the floor must be # [40, ~80]. A genuine in-coverage reading at or just above the floor must be
# PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A # PRESERVED, not clamped UP to the floor. Only true-quiet 0.0 is floored. A
# quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99 # quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
# dB readings; this pins that they survive unchanged. # dB readings; this pins that they survive unchanged.
floor = float(noise.NOISE_QUIET_FLOOR_DB) floor = float(noise.NOISE_QUIET_FLOOR_DB)

View file

@ -50,7 +50,7 @@ ENGLAND_PBF_URL = (
"https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf" "https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf"
) )
# Bus Open Data Service pre-converted GTFS covering all England bus/tram/ferry # Bus Open Data Service: pre-converted GTFS covering all England bus/tram/ferry
BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/" BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
# National Rail Open Data API # National Rail Open Data API
@ -597,7 +597,7 @@ def validate_gtfs_feed(
fail("has neither calendar.txt nor calendar_dates.txt") fail("has neither calendar.txt nor calendar_dates.txt")
if not _calendar_active_in_window(z, names, window_start, window_end): if not _calendar_active_in_window(z, names, window_start, window_end):
fail( fail(
f"no service active between {window_start} and {window_end} " f"no service active between {window_start} and {window_end}: "
"the feed's calendars are stale/expired and it would contribute " "the feed's calendars are stale/expired and it would contribute "
"zero service to routing" "zero service to routing"
) )
@ -697,7 +697,7 @@ def _iter_stop_time_trips(lines, trip_id_idx: int):
dtd2mysql currently writes rows grouped by trip and ordered by dtd2mysql currently writes rows grouped by trip and ordered by
stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a
trip_id reappearing later raises instead of silently scrambling trips); trip_id reappearing later raises instead of silently scrambling trips);
within-trip order is NOT assumed callers sort each group by its original within-trip order is NOT assumed: callers sort each group by its original
stop_sequence. stop_sequence.
""" """
current_trip: str | None = None current_trip: str | None = None
@ -1175,7 +1175,7 @@ def main() -> None:
cif = download_national_rail_cif(raw_dir) cif = download_national_rail_cif(raw_dir)
if cif is None: if cif is None:
raise RuntimeError( raise RuntimeError(
"National Rail timetable was not downloaded set " "National Rail timetable was not downloaded: set "
"NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at " "NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at "
"https://opendata.nationalrail.co.uk/). National Rail heavy rail is " "https://opendata.nationalrail.co.uk/). National Rail heavy rail is "
"required; without it the transit network models every train journey " "required; without it the transit network models every train journey "

View file

@ -1,8 +1,8 @@
"""Robust GEOS overlay helpers. """Robust GEOS overlay helpers.
Overlay operations (union, difference, intersection) can raise a Overlay operations (union, difference, intersection) can raise a
``GEOSException`` most often ``TopologyException: side location conflict``, ``GEOSException``, most often ``TopologyException: side location conflict``,
``Ring edge missing``, or ``found non-noded intersection`` on geometries that ``Ring edge missing``, or ``found non-noded intersection``, on geometries that
contain near-coincident or near-degenerate edges, or that are individually contain near-coincident or near-degenerate edges, or that are individually
invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG
engine, handed a grid size, nodes every edge onto that grid and finishes where engine, handed a grid size, nodes every edge onto that grid and finishes where
@ -14,8 +14,8 @@ learned the hard way from a crash:
1. **Never precision-reduce with the default mode.** ``set_precision``'s default 1. **Never precision-reduce with the default mode.** ``set_precision``'s default
``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that ``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that
re-raises the very ``side location conflict`` we are trying to escape. We push re-raises the very ``side location conflict`` we are trying to escape. We push
the grid into the overlay via the ``grid_size`` argument instead where the grid into the overlay via the ``grid_size`` argument instead (where
OverlayNG nodes robustly and only ever call ``set_precision`` in OverlayNG nodes robustly) and only ever call ``set_precision`` in
``pointwise`` mode (pure coordinate rounding, which cannot raise). ``pointwise`` mode (pure coordinate rounding, which cannot raise).
2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties, 2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties,
pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape. pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape.
@ -40,7 +40,7 @@ from shapely import GEOSException, make_valid, set_precision
from shapely.geometry import Polygon from shapely.geometry import Polygon
from shapely.ops import unary_union from shapely.ops import unary_union
# 0.1 mm in metres well below MIN_GEOM_AREA (0.01 m^2) and survey resolution. # 0.1 mm in metres: well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
_SNAP_GRID = 1e-4 _SNAP_GRID = 1e-4
_EMPTY = Polygon() _EMPTY = Polygon()

View file

@ -120,7 +120,7 @@ def _is_pointlike(geom_bng) -> bool:
def _rescue_footprint(geom_bng) -> dict | None: def _rescue_footprint(geom_bng) -> dict | None:
"""Fatten a degenerate BNG geometry into a representable footprint and snap. """Fatten a degenerate BNG geometry into a representable footprint and snap.
A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon the A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon, the
signature of a tower-block postcode whose UPRNs all share one coordinate) signature of a tower-block postcode whose UPRNs all share one coordinate)
gets a building-scale buffer so it is not reduced to an invisible sub-metre gets a building-scale buffer so it is not reduced to an invisible sub-metre
dot; thin slivers that still carry length keep the minimal buffer. dot; thin slivers that still carry length keep the minimal buffer.
@ -263,7 +263,7 @@ def merge_fragments(
# Close tiny gaps between adjacent OA boundary edges (float mismatches). # Close tiny gaps between adjacent OA boundary edges (float mismatches).
# The closing can erode a tiny MultiPolygon (e.g. a postcode with only a # The closing can erode a tiny MultiPolygon (e.g. a postcode with only a
# sliver fragment) to nothing, which would leave the postcode with no # sliver fragment) to nothing, which would leave the postcode with no
# geometry at all — keep the un-closed shape if that happens. # geometry at all. Keep the un-closed shape if that happens.
if combined.geom_type == "MultiPolygon": if combined.geom_type == "MultiPolygon":
closed = combined.buffer(5.0).buffer(-5.0) closed = combined.buffer(5.0).buffer(-5.0)
if not closed.is_valid: if not closed.is_valid:
@ -308,7 +308,7 @@ def _polygonal(geom):
return None return None
# Both callers run on WGS84-degree output geometry, so the robustness # Both callers run on WGS84-degree output geometry, so the robustness
# fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre # fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre
# default — a coarse metre grid would obliterate a degree-scale shape. # default. A coarse metre grid would obliterate a degree-scale shape.
merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG) merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG)
return merged if not merged.is_empty else None return merged if not merged.is_empty else None
return None return None
@ -324,7 +324,7 @@ def _resolve_overlaps(
containment (a postcode fully enclosed by another). Each postcode is trimmed containment (a postcode fully enclosed by another). Each postcode is trimmed
by the union of its higher-priority overlapping neighbours, where **priority = by the union of its higher-priority overlapping neighbours, where **priority =
ascending area**: a smaller postcode wins contested ground. That single rule ascending area**: a smaller postcode wins contested ground. That single rule
handles both cases correctly an enclosed postcode is always smaller than its handles both cases correctly: an enclosed postcode is always smaller than its
container, so it keeps its area while the container gets a hole (a `overlaps` container, so it keeps its area while the container gets a hole (a `overlaps`
query alone would miss containment entirely). Run last, on the final output query alone would miss containment entirely). Run last, on the final output
geometries, so nothing re-introduces overlap afterwards. A postcode that would geometries, so nothing re-introduces overlap afterwards. A postcode that would
@ -348,7 +348,7 @@ def _resolve_overlaps(
arr = np.array(geoms, dtype=object) arr = np.array(geoms, dtype=object)
pairs: set[tuple[int, int]] = set() pairs: set[tuple[int, int]] = set()
# "overlaps" gives partial overlaps; "contains" gives containment (which # "overlaps" gives partial overlaps; "contains" gives containment (which
# "overlaps" excludes) — together they cover every 2-D overlap without the # "overlaps" excludes). Together they cover every 2-D overlap without the
# edge-touch explosion a plain "intersects" query would add. # edge-touch explosion a plain "intersects" query would add.
for predicate in ("overlaps", "contains"): for predicate in ("overlaps", "contains"):
qsrc, qtgt = tree.query(arr, predicate=predicate) qsrc, qtgt = tree.query(arr, predicate=predicate)
@ -577,7 +577,7 @@ def _grid_footprint(geom):
pass can shave a small (e.g. co-located, non-geographic) postcode down to a pass can shave a small (e.g. co-located, non-geographic) postcode down to a
sub-grid sliver that disappears when snapped to output precision. Rather than sub-grid sliver that disappears when snapped to output precision. Rather than
drop it, place a minimal valid footprint at its location. The tiny overlap drop it, place a minimal valid footprint at its location. The tiny overlap
this re-creates with the neighbour that trimmed it is harmless the output this re-creates with the neighbour that trimmed it is harmless: the output
partition is best-effort, a missing boundary is a hard validation failure. partition is best-effort, a missing boundary is a hard validation failure.
""" """
try: try:

View file

@ -11,7 +11,7 @@ from .voronoi import compute_voronoi_regions
MIN_GEOM_AREA = 0.01 MIN_GEOM_AREA = 0.01
# Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a # Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a
# crowded multi-postcode OA its Voronoi cell ∩ remaining collapses below # crowded multi-postcode OA: its Voronoi cell ∩ remaining collapses below
# MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a # MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a
# co-located postcode. Every *active* postcode must keep a boundary # co-located postcode. Every *active* postcode must keep a boundary
# (validate_outputs is zero-tolerance), so it gets a small disc at its true seed # (validate_outputs is zero-tolerance), so it gets a small disc at its true seed
@ -77,7 +77,7 @@ def process_oa(
fragments.append((pc, merged)) fragments.append((pc, merged))
# Every postcode with a UPRN seed in this OA must keep at least a minimal # Every postcode with a UPRN seed in this OA must keep at least a minimal
# footprint in a dense OA (a block of flats with hundreds of distinct # footprint: in a dense OA (a block of flats with hundreds of distinct
# postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or # postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or
# be fully absorbed by a co-located postcode's INSPIRE parcel, producing no # be fully absorbed by a co-located postcode's INSPIRE parcel, producing no
# fragment, and an active postcode must never be dropped. # fragment, and an active postcode must never be dropped.
@ -142,7 +142,7 @@ def _claim_inspire_parcels(
# UPRNs from a single postcode goes wholly to that postcode. A parcel shared # UPRNs from a single postcode goes wholly to that postcode. A parcel shared
# by several postcodes (a block of flats spanning postcodes, or overlapping # by several postcodes (a block of flats spanning postcodes, or overlapping
# parcel data) is split between them via a sub-Voronoi over their own UPRNs # parcel data) is split between them via a sub-Voronoi over their own UPRNs
# clipped to the parcel so EVERY contained postcode keeps part of the # clipped to the parcel, so EVERY contained postcode keeps part of the
# parcel. A bare majority vote would hand the whole parcel to one winner and # parcel. A bare majority vote would hand the whole parcel to one winner and
# leave the losers' UPRNs trapped inside claimed land, dropping them from # leave the losers' UPRNs trapped inside claimed land, dropping them from
# both this claim and the `remaining` polygon handed to Voronoi downstream. # both this claim and the `remaining` polygon handed to Voronoi downstream.
@ -312,7 +312,7 @@ def _extract_polygonal(geom) -> Polygon | MultiPolygon | None:
return polys[0] return polys[0]
# Union (not bare MultiPolygon construction): make_valid can emit # Union (not bare MultiPolygon construction): make_valid can emit
# overlapping polygonal parts, and a MultiPolygon of overlapping parts is # overlapping polygonal parts, and a MultiPolygon of overlapping parts is
# invalid it double-counts area and makes the next `.difference()` raise # invalid: it double-counts area and makes the next `.difference()` raise
# a TopologyException that aborts the OA (and, in parallel mode, the # a TopologyException that aborts the OA (and, in parallel mode, the
# worker). safe_union merges them into a valid geometry. # worker). safe_union merges them into a valid geometry.
merged = safe_union(polys) merged = safe_union(polys)

View file

@ -82,7 +82,7 @@ def load_uprns(
# Remap terminated postcodes to their nearest active successor. The # Remap terminated postcodes to their nearest active successor. The
# successor generally lives in a DIFFERENT OA (and at different grid # successor generally lives in a DIFFERENT OA (and at different grid
# coordinates), so the remapped point must adopt the successor's # coordinates), so the remapped point must adopt the successor's
# authoritative OA/coords — keeping the terminated postcode's original # authoritative OA/coords. Keeping the terminated postcode's original
# OA would seed the successor into an OA it doesn't belong to, splitting # OA would seed the successor into an OA it doesn't belong to, splitting
# its boundary across OAs. Genuine (non-remapped) UPRN rows keep their # its boundary across OAs. Genuine (non-remapped) UPRN rows keep their
# own OA, since a live postcode can legitimately span several OAs. # own OA, since a live postcode can legitimately span several OAs.
@ -127,7 +127,7 @@ def load_uprns(
uprns.sort("OA21CD").sink_parquet(tmp_path) uprns.sort("OA21CD").sink_parquet(tmp_path)
release_memory() release_memory()
# Read the sorted data only one copy in memory (~2GB) # Read the sorted data: only one copy in memory (~2GB)
df = pl.read_parquet(tmp_path) df = pl.read_parquet(tmp_path)
tmp_path.unlink() tmp_path.unlink()
n = len(df) n = len(df)

View file

@ -4,7 +4,7 @@ Stratified by property type and postcode sector, with IRLS Huber regression,
hierarchical shrinkage (sector district area national hedonic), hierarchical shrinkage (sector district area national hedonic),
and KD-tree spatial smoothing for sparse sectors. and KD-tree spatial smoothing for sparse sectors.
Output: price_index.parquet sector x type_group x year -> log_index Output: price_index.parquet (sector x type_group x year -> log_index)
""" """
import argparse import argparse

View file

@ -23,7 +23,7 @@ ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
# Natural join key from estimates back onto properties: postcode plus the # Natural join key from estimates back onto properties: postcode plus the
# coalesced register/EPC address. This is unique and non-null on the deduped # coalesced register/EPC address. This is unique and non-null on the deduped
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps # dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
# estimates 1:1 onto properties regardless of row order — estimates are computed # estimates 1:1 onto properties regardless of row order. Estimates are computed
# from a separate price_inputs.parquet, so a positional key would not line up. # from a separate price_inputs.parquet, so a positional key would not line up.
JOIN_ADDRESS = "_join_address" JOIN_ADDRESS = "_join_address"
JOIN_KEYS = ["Postcode", JOIN_ADDRESS] JOIN_KEYS = ["Postcode", JOIN_ADDRESS]

View file

@ -315,7 +315,7 @@ def test_run_tenure_history_tracks_rent_owner_transitions(tmp_path: Path):
def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path): def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
# A property only ever observed as owner-occupied has no tenure change worth # A property only ever observed as owner-occupied has no tenure change worth
# surfacing the timeline column is null (no events), not a noisy baseline. # surfacing: the timeline column is null (no events), not a noisy baseline.
zip_path = tmp_path / "domestic-csv.zip" zip_path = tmp_path / "domestic-csv.zip"
_write_epc_zip( _write_epc_zip(
zip_path, zip_path,
@ -339,6 +339,37 @@ def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
assert df.get_column("tenure_history").to_list() == [None] assert df.get_column("tenure_history").to_list() == [None]
def test_run_latest_tenure_status_reflects_most_recent_certificate(tmp_path: Path):
# Two certificates for one dwelling: an older social-rented cert and a newer
# owner-occupied one. The published latest_tenure_status must carry the
# LATEST certificate's normalized tenure ("Owner-occupied"), while
# was_council_house stays "Yes" because the dwelling was social at some
# point. This also confirms latest_tenure_status reaches the epc_pp parquet.
zip_path = tmp_path / "domestic-csv.zip"
_write_epc_zip(
zip_path,
[
_row(inspection_date="2016-04-01", tenure="Rented (social)"),
_row(inspection_date="2024-04-01", tenure="owner-occupied"),
],
)
price_paid_path = tmp_path / "price-paid.parquet"
_price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet(
price_paid_path
)
output_path = tmp_path / "epc-pp.parquet"
_run(zip_path, price_paid_path, output_path, tmp_path)
df = pl.read_parquet(output_path)
assert df.height == 1
assert df.select("latest_tenure_status", "was_council_house").to_dicts() == [
{"latest_tenure_status": "Owner-occupied", "was_council_house": "Yes"}
]
def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path): def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path):
# Two certificates for the same property. The cert with the garbled, # Two certificates for the same property. The cert with the garbled,
# unparseable inspection_date must NOT be chosen as "latest": a string sort # unparseable inspection_date must NOT be chosen as "latest": a string sort
@ -565,7 +596,7 @@ def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: P
price_paid_path = tmp_path / "price-paid.parquet" price_paid_path = tmp_path / "price-paid.parquet"
pl.DataFrame( pl.DataFrame(
{ {
# 5_000 is below MIN_PRICE (10_000) a nominal/junk transfer that # 5_000 is below MIN_PRICE (10_000), a nominal/junk transfer that
# must still anchor the construction year but stay out of the price # must still anchor the construction year but stay out of the price
# aggregations. # aggregations.
"price": [5_000, 300_000], "price": [5_000, 300_000],
@ -603,7 +634,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
# lands AFTER its first Land Registry sale (1998). A dwelling cannot have # lands AFTER its first Land Registry sale (1998). A dwelling cannot have
# been built after it was first sold, so the published build year must be # been built after it was first sold, so the published build year must be
# capped at the first transfer year (1998), not the later band estimate. # capped at the first transfer year (1998), not the later band estimate.
# It stays flagged as an estimate (approximate=1) — it is still EPC-derived. # It stays flagged as an estimate (approximate=1). It is still EPC-derived.
zip_path = tmp_path / "domestic-csv.zip" zip_path = tmp_path / "domestic-csv.zip"
_write_epc_zip( _write_epc_zip(
zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")] zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")]
@ -627,7 +658,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path): def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
# The common case: the EPC band (1950-1966 -> 1958) predates the first # The common case: the EPC band (1950-1966 -> 1958) predates the first
# recorded sale (2020). The cap must NOT fire — the band estimate stands. # recorded sale (2020). The cap must NOT fire. The band estimate stands.
zip_path = tmp_path / "domestic-csv.zip" zip_path = tmp_path / "domestic-csv.zip"
_write_epc_zip(zip_path) _write_epc_zip(zip_path)
@ -649,7 +680,7 @@ def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path): def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
# A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the # A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the
# NEW floor (10k): it must now be RETAINED in the price aggregations. This # NEW floor (10k): it must now be RETAINED in the price aggregations. This
# pins the 50k->10k change it fails on the pre-fix 50k floor (where 30k was # pins the 50k->10k change: it fails on the pre-fix 50k floor (where 30k was
# excluded, giving historical_prices length 1 / latest_price 250_000). # excluded, giving historical_prices length 1 / latest_price 250_000).
zip_path = tmp_path / "domestic-csv.zip" zip_path = tmp_path / "domestic-csv.zip"
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
@ -748,8 +779,8 @@ def test_run_collapses_duplicate_transactions(tmp_path: Path):
# The duplicated 250_000 sale collapses to one entry; two distinct sales. # The duplicated 250_000 sale collapses to one entry; two distinct sales.
assert df.get_column("historical_prices").to_list() == [ assert df.get_column("historical_prices").to_list() == [
[ [
{"year": 2020, "month": 2, "price": 200_000}, {"year": 2020, "month": 2, "price": 200_000, "is_new": False},
{"year": 2024, "month": 2, "price": 250_000}, {"year": 2024, "month": 2, "price": 250_000, "is_new": False},
] ]
] ]
assert df.get_column("latest_price").to_list() == [250_000] assert df.get_column("latest_price").to_list() == [250_000]
@ -777,7 +808,7 @@ def test_run_excludes_implausible_price_jump_but_keeps_property(tmp_path: Path):
assert df.height == 1 assert df.height == 1
assert df.get_column("latest_price").to_list() == [140_000] assert df.get_column("latest_price").to_list() == [140_000]
assert df.get_column("historical_prices").to_list() == [ assert df.get_column("historical_prices").to_list() == [
[{"year": 2016, "month": 6, "price": 140_000}] [{"year": 2016, "month": 6, "price": 140_000, "is_new": False}]
] ]

View file

@ -593,7 +593,7 @@ def test_transform_grocery_dedup_drops_only_grocery_aspect(tmp_path):
# The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at # The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at
# (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name # (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name
# is the same physical store, so its Convenience Store (Groceries) row is a # is the same physical store, so its Convenience Store (Groceries) row is a
# duplicate and must be dropped but its NON-grocery aspect (a Post Office # duplicate and must be dropped, but its NON-grocery aspect (a Post Office
# sharing the same OSM id) must survive. An independent shop away from the # sharing the same OSM id) must survive. An independent shop away from the
# GEOLYTIX point keeps its grocery row. # GEOLYTIX point keeps its grocery row.
raw = pl.DataFrame( raw = pl.DataFrame(

View file

@ -34,7 +34,7 @@ DROP_CATEGORIES = {
"emergency/water_tank", "emergency/water_tank",
"leisure/bleachers", "leisure/bleachers",
"leisure/schoolyard", "leisure/schoolyard",
# Park "furniture" / incidental features not parks; they massively # Park "furniture" / incidental features, not parks; they massively
# inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k). # inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k).
"leisure/bandstand", "leisure/bandstand",
"leisure/bird_hide", "leisure/bird_hide",
@ -222,7 +222,7 @@ DROP_CATEGORIES = {
"public_transport/entrance", "public_transport/entrance",
"public_transport/station", "public_transport/station",
"public_transport/stop_position", "public_transport/stop_position",
# Education amenities — schools come from GIAS instead. OSM coverage for # Education amenities. Schools come from GIAS instead. OSM coverage for
# tertiary education, tutoring, and childcare is too noisy/incomplete to be # tertiary education, tutoring, and childcare is too noisy/incomplete to be
# useful on a property-search map. # useful on a property-search map.
"amenity/school", "amenity/school",
@ -398,7 +398,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
"tourism/theme_park", "tourism/theme_park",
# bicycle_rental/boat_rental/marina/slipway used to live here and # bicycle_rental/boat_rental/marina/slipway used to live here and
# made up ~46% of the bucket (cycle-hire docks, boat ramps); they # made up ~46% of the bucket (cycle-hire docks, boat ramps); they
# are infrastructure, not entertainment venues — see DROP_CATEGORIES. # are infrastructure, not entertainment venues. See DROP_CATEGORIES.
"leisure/hackerspace", "leisure/hackerspace",
"leisure/yes", "leisure/yes",
], ],
@ -722,7 +722,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
[ [
"leisure/fitness_centre", "leisure/fitness_centre",
# leisure/fitness_station (outdoor pull-up bars / trim-trail # leisure/fitness_station (outdoor pull-up bars / trim-trail
# apparatus, ~2.5k) is not a gym — see DROP_CATEGORIES. # apparatus, ~2.5k) is not a gym. See DROP_CATEGORIES.
"amenity/dojo", "amenity/dojo",
"amenity/dancing_school", "amenity/dancing_school",
], ],
@ -849,8 +849,8 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
"healthcare/pharmacy", "healthcare/pharmacy",
"shop/chemist", "shop/chemist",
# healthcare/alternative, shop/herbalist and shop/health (homeopaths, # healthcare/alternative, shop/herbalist and shop/health (homeopaths,
# herbalists, generic "health" shops) are not dispensing pharmacies # herbalists, generic "health" shops) are not dispensing pharmacies.
# — see DROP_CATEGORIES. # See DROP_CATEGORIES.
], ],
), ),
# "Hospital & Clinic" used to be one bucket; an actual hospital and a small # "Hospital & Clinic" used to be one bucket; an actual hospital and a small
@ -878,7 +878,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
"healthcare/laboratory", "healthcare/laboratory",
"healthcare/rehabilitation", "healthcare/rehabilitation",
"healthcare/vaccination_centre", "healthcare/vaccination_centre",
# healthcare/yes (untyped junk rows) is dropped — see DROP_CATEGORIES. # healthcare/yes (untyped junk rows) is dropped. See DROP_CATEGORIES.
], ],
), ),
( (
@ -950,7 +950,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
[ [
"tourism/gallery", "tourism/gallery",
# tourism/artwork (statues, murals, village signs) was 93% of this # tourism/artwork (statues, murals, village signs) was 93% of this
# bucket and is not a visitable gallery — see DROP_CATEGORIES. # bucket and is not a visitable gallery. See DROP_CATEGORIES.
], ],
), ),
( (
@ -1420,7 +1420,7 @@ def _school_icon_category_expr() -> pl.Expr:
# GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed # GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed
# primary") so we normalise before matching. # primary") so we normalise before matching.
phase = pl.col("phase").str.to_lowercase() phase = pl.col("phase").str.to_lowercase()
# gias._format_age_range emits three shapes: "<low><high>" (em-dash), # gias._format_age_range emits three shapes: "<low><high>" (en-dash),
# "up to <high>" (high-only) and "<low>+" (low-only). Extract the leading # "up to <high>" (high-only) and "<low>+" (low-only). Extract the leading
# integer as low and the trailing integer as high, then suppress the wrong # integer as low and the trailing integer as high, then suppress the wrong
# end for the one-sided shapes so they don't collapse to a single bound. # end for the one-sided shapes so they don't collapse to a single bound.
@ -1477,7 +1477,7 @@ def _load_ofsted_ratings(ofsted_path: Path) -> pl.LazyFrame:
the conventional Ofsted labels; when there is no usable graded result the conventional Ofsted labels; when there is no usable graded result
(null/"Not judged", e.g. schools last seen under the post-2024 ungraded (null/"Not judged", e.g. schools last seen under the post-2024 ungraded
report-card framework) we fall back to "Ungraded inspection overall outcome" report-card framework) we fall back to "Ungraded inspection overall outcome"
so genuinely good/outstanding schools aren't dropped mirroring so genuinely good/outstanding schools aren't dropped, mirroring
school_catchments.classify_good_plus_schools. Remaining nulls drop out.""" school_catchments.classify_good_plus_schools. Remaining nulls drop out."""
grade_col = pl.col("Latest OEIF overall effectiveness") grade_col = pl.col("Latest OEIF overall effectiveness")
# See school_catchments: the ungraded outcome carries "School remains Good"/ # See school_catchments: the ungraded outcome carries "School remains Good"/
@ -1682,7 +1682,7 @@ def osm_groceries_colocated_with_geolytix(
An OSM Groceries row is a duplicate when a GEOLYTIX point lies within An OSM Groceries row is a duplicate when a GEOLYTIX point lies within
``radius_m`` metres AND that point's brand tokens (its ``category``, e.g. ``radius_m`` metres AND that point's brand tokens (its ``category``, e.g.
"Tesco", "Co-op", "M&S") are all present in the OSM row's name i.e. the "Tesco", "Co-op", "M&S") are all present in the OSM row's name, i.e. the
same physical branded store. Short brands like "M&S" match via same physical branded store. Short brands like "M&S" match via
_GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept. _GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept.
@ -1703,7 +1703,7 @@ def osm_groceries_colocated_with_geolytix(
osm_ids = osm_groceries["id"].to_list() osm_ids = osm_groceries["id"].to_list()
osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()] osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()]
# Equirectangular projection to metres around the shared mean latitude — at # Equirectangular projection to metres around the shared mean latitude. At
# England's scale this is accurate to well under the dedup radius. # England's scale this is accurate to well under the dedup radius.
mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat]))) mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat])))
cos_lat = float(np.cos(np.radians(mean_lat))) cos_lat = float(np.cos(np.radians(mean_lat)))
@ -1860,7 +1860,7 @@ def transform(
) )
# Scope the drop to the Groceries group: a single OSM object can also # Scope the drop to the Groceries group: a single OSM object can also
# carry a non-grocery aspect (e.g. a convenience store that is also a # carry a non-grocery aspect (e.g. a convenience store that is also a
# Post Office), which must survive — only its duplicate grocery row goes. # Post Office), which must survive. Only its duplicate grocery row goes.
lf = lf.filter( lf = lf.filter(
~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids)) ~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids))
) )

View file

@ -45,7 +45,7 @@ POSTCODE_DENSITY_PERCENTILE_COL = "Tree canopy density percentile within {radius
POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)" POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)"
POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)" POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)"
# National Forest Inventory (NFI) woodland the geometric complement of TOW. # National Forest Inventory (NFI) woodland: the geometric complement of TOW.
# NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700. # NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700.
# Field names are from the NFI Woodland England 2022 release; re-check on bumps. # Field names are from the NFI Woodland England 2022 release; re-check on bumps.
NFI_CATEGORY_COL = "CATEGORY" NFI_CATEGORY_COL = "CATEGORY"
@ -263,7 +263,7 @@ def _postcode_buffers(
return circles, shapely.STRtree(circles) return circles, shapely.STRtree(circles)
# 0.1 mm in the BNG working CRS (EPSG:27700) far below survey resolution; the # 0.1 mm in the BNG working CRS (EPSG:27700), far below survey resolution; the
# same grid the postcode_boundaries overlay uses. # same grid the postcode_boundaries overlay uses.
_OVERLAY_GRID_M = 1e-4 _OVERLAY_GRID_M = 1e-4
@ -274,13 +274,13 @@ def _robust_intersection_area(a: np.ndarray, b: np.ndarray) -> np.ndarray:
External Forest Research TOW/NFI polygons are occasionally invalid External Forest Research TOW/NFI polygons are occasionally invalid
(self-intersections), and a single bad polygon makes the batched (self-intersections), and a single bad polygon makes the batched
``shapely.intersection`` raise ``TopologyException: side location conflict``, ``shapely.intersection`` raise ``TopologyException: side location conflict``,
aborting the whole run. The fast path is the raw batched overlay unchanged, aborting the whole run. The fast path is the raw batched overlay (unchanged,
full-speed, when the data is clean and only a failure triggers repair. full-speed, when the data is clean), and only a failure triggers repair.
The repair deliberately uses a *plain* overlay rather than the fixed-precision The repair deliberately uses a *plain* overlay rather than the fixed-precision
(``grid_size``) one: ``make_valid`` can emit a mixed-dimension (``grid_size``) one: ``make_valid`` can emit a mixed-dimension
``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG ``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG
rejects with ``Overlay input is mixed-dimension`` whereas a plain overlay rejects with ``Overlay input is mixed-dimension``, whereas a plain overlay
accepts it, and its non-polygonal debris has zero area and is dropped by the accepts it, and its non-polygonal debris has zero area and is dropped by the
``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate ``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate
snap (which never raises) collapses the near-coincident edges behind any snap (which never raises) collapses the near-coincident edges behind any
@ -496,7 +496,7 @@ def _finalize_metrics(
if over_count: if over_count:
print( print(
f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were " f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were "
"capped — indicates overlapping TOW/NFI canopy within the buffer" "capped, indicating overlapping TOW/NFI canopy within the buffer"
) )
mean_height = np.divide( mean_height = np.divide(

View file

@ -92,7 +92,7 @@ def test_custom_radius(pois):
} }
) )
# 0.01 km = 10m only the POI at the exact same location should match # 0.01 km = 10m: only the POI at the exact same location should match
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01) result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01)
# The Restaurant at (51.5074, -0.1278) is at distance 0 # The Restaurant at (51.5074, -0.1278) is at distance 0
assert result["restaurants_0km"][0] >= 1 assert result["restaurants_0km"][0] >= 1
@ -135,9 +135,9 @@ def test_min_distance_finds_nearest(postcodes, pois):
assert len(result) == 2 assert len(result) == 2
ec1a = result.filter(pl.col("postcode") == "EC1A 1BB") ec1a = result.filter(pl.col("postcode") == "EC1A 1BB")
# Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278) very close # Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278), very close
assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m
# Restaurant is co-located distance ~0 # Restaurant is co-located: distance ~0
assert ec1a["restaurants_nearest_km"][0] < 0.01 assert ec1a["restaurants_nearest_km"][0] < 0.01
# Far-away postcode should still get the global nearest distance. # Far-away postcode should still get the global nearest distance.