.
This commit is contained in:
parent
909e241907
commit
1ee796b282
29 changed files with 250 additions and 126 deletions
|
|
@ -14,7 +14,7 @@ def _prop(pid, pin=None):
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _needs_detail_fetch — accurate-pin skip
|
||||
# _needs_detail_fetch: accurate-pin skip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -32,7 +32,7 @@ def test_needs_detail_fetch_disabled_always_fetches(monkeypatch):
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _prime_detail_postcodes — worklist selection + concurrent fetch
|
||||
# _prime_detail_postcodes: worklist selection + concurrent fetch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -97,7 +97,7 @@ def test_prime_is_a_noop_when_disabled_or_cap_zero(monkeypatch):
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _paginate — end-to-end (network stubbed): accurate pins fall back to
|
||||
# _paginate end-to-end (network stubbed): accurate pins fall back to
|
||||
# coordinates, approximate pins use the detail postcode.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import zoopla
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _run_sources — Zoopla inline, others in threads, failures isolated
|
||||
# _run_sources: Zoopla inline, others in threads, failures isolated
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -85,13 +85,13 @@ def test_seed_and_save_detail_caches_round_trip(tmp_path):
|
|||
|
||||
def test_seed_detail_caches_tolerates_missing_files(tmp_path):
|
||||
rightmove._detail_postcode_cache.clear()
|
||||
# No file written yet — seeding must not raise and must leave cache empty.
|
||||
# No file written yet: seeding must not raise and must leave cache empty.
|
||||
scraper._seed_detail_caches(["rightmove"], tmp_path)
|
||||
assert rightmove._detail_postcode_cache == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_scrape — full orchestration wiring (sources stubbed, no network)
|
||||
# run_scrape: full orchestration wiring (sources stubbed, no network)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from transform import (
|
||||
build_listing_url,
|
||||
build_register_address,
|
||||
clean_listing_address,
|
||||
extract_full_postcode,
|
||||
|
|
@ -173,3 +174,33 @@ def test_rightmove_transform_without_detail_keeps_coordinate_logic() -> None:
|
|||
assert result is not None
|
||||
assert result["Postcode"] == "SW9 7AA"
|
||||
assert result["Postcode source"] == "coordinates"
|
||||
|
||||
|
||||
def test_build_listing_url_stamps_channel_from_new_build_flag() -> None:
|
||||
# Resale gets RES_BUY; new builds get RES_NEW.
|
||||
assert build_listing_url("/properties/200", False) == (
|
||||
"https://www.rightmove.co.uk/properties/200#/?channel=RES_BUY"
|
||||
)
|
||||
assert build_listing_url("/properties/200", True) == (
|
||||
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
|
||||
)
|
||||
# An existing channel/fragment on the source URL is stripped and re-stamped.
|
||||
assert build_listing_url("/properties/200#/?channel=RES_BUY", True) == (
|
||||
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
|
||||
)
|
||||
# Missing URL stays empty.
|
||||
assert build_listing_url("", True) == ""
|
||||
|
||||
|
||||
def test_rightmove_transform_tags_new_builds_res_new() -> None:
|
||||
# The Rightmove search response marks new-build developments with
|
||||
# development=True; transform_property must stamp the listing URL RES_NEW.
|
||||
new_build = {**_rightmove_prop(), "development": True}
|
||||
result = transform_property(new_build, "SW9", StubPostcodeIndex("SW9 7AA"))
|
||||
assert result is not None
|
||||
assert result["Listing URL"].endswith("#/?channel=RES_NEW")
|
||||
|
||||
# Ordinary resale (development absent/false) stays RES_BUY.
|
||||
resale = transform_property(_rightmove_prop(), "SW9", StubPostcodeIndex("SW9 7AA"))
|
||||
assert resale is not None
|
||||
assert resale["Listing URL"].endswith("#/?channel=RES_BUY")
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ def test_parse_detail_geo_merges_location_uprn_with_address_full_address() -> No
|
|||
def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None:
|
||||
# The only `address` twin on the page belongs to a different uprn (a
|
||||
# comparable listing). With a uprn to match on, an unrelated twin is never
|
||||
# borrowed — full_address stays None rather than grabbing the wrong street.
|
||||
# borrowed: full_address stays None rather than grabbing the wrong street.
|
||||
html = (
|
||||
'"location":{"outcode":"NR29",'
|
||||
'"coordinates":{"latitude":52.716014,"longitude":1.614495},'
|
||||
|
|
@ -185,7 +185,7 @@ def test_parse_detail_geo_returns_none_for_garbage() -> None:
|
|||
assert parse_detail_geo("<html><body>no data here</body></html>") is None
|
||||
assert parse_detail_geo("") is None
|
||||
# Coordinates that are not inside a property location/address wrapper (e.g.
|
||||
# only an unwrapped POI) yield nothing — safe degradation to the outcode.
|
||||
# only an unwrapped POI) yield nothing, safe degradation to the outcode.
|
||||
assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ def map_property_type(sub_type: str | None) -> str:
|
|||
return "Terraced"
|
||||
if "house" in lower or "cottage" in lower:
|
||||
return "Detached"
|
||||
log.warning("Unknown propertySubType: %r — mapping to Other", sub_type)
|
||||
log.warning("Unknown propertySubType: %r, mapping to Other", sub_type)
|
||||
return "Other"
|
||||
|
||||
|
||||
|
|
@ -267,7 +267,7 @@ def build_register_address(
|
|||
property's own number or name (e.g. Zoopla detail pages expose
|
||||
``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address
|
||||
carries the house identifier that the EPC/Price-Paid register addresses also
|
||||
use — turning a fuzzy street match into a near-exact one. Falls back to the
|
||||
use, turning a fuzzy street match into a near-exact one. Falls back to the
|
||||
plain cleaned address when no number/name is available.
|
||||
"""
|
||||
cleaned = clean_listing_address(raw_address)
|
||||
|
|
@ -282,6 +282,23 @@ def build_register_address(
|
|||
return f"{number_or_name}, {cleaned}" if cleaned else number_or_name
|
||||
|
||||
|
||||
def build_listing_url(property_url: str | None, is_new_build: bool) -> str:
|
||||
"""Build the canonical Rightmove listing URL with an explicit channel marker.
|
||||
|
||||
The search API always echoes ``?channel=RES_BUY`` in ``propertyUrl`` even for
|
||||
new-build developments (the request channel stays BUY), so the channel is
|
||||
re-stamped here from the per-listing ``development`` flag: ``RES_NEW`` for new
|
||||
builds, ``RES_BUY`` for ordinary resale. The map UI reads this marker to split
|
||||
new vs non-new listings. Any channel/fragment already on ``propertyUrl`` is
|
||||
stripped first so the result is deterministic.
|
||||
"""
|
||||
if not property_url:
|
||||
return ""
|
||||
base = property_url.split("#", 1)[0].split("?", 1)[0]
|
||||
channel = "RES_NEW" if is_new_build else "RES_BUY"
|
||||
return f"{RIGHTMOVE_BASE}{base}#/?channel={channel}"
|
||||
|
||||
|
||||
def transform_property(
|
||||
prop: dict,
|
||||
outcode: str,
|
||||
|
|
@ -337,7 +354,7 @@ def transform_property(
|
|||
|
||||
inferred_postcode = pc_index.nearest(lat, lng)
|
||||
if not inferred_postcode:
|
||||
log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng)
|
||||
log.debug("No England postcode for property at %.4f, %.4f; skipping", lat, lng)
|
||||
return None
|
||||
raw_address = prop.get("displayAddress", "") or ""
|
||||
extracted_postcode = extract_full_postcode(raw_address)
|
||||
|
|
@ -391,7 +408,7 @@ def transform_property(
|
|||
"price_frequency": "",
|
||||
"Price qualifier": price_qualifier,
|
||||
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
|
||||
"Listing URL": RIGHTMOVE_BASE + property_url if property_url else "",
|
||||
"Listing URL": build_listing_url(property_url, bool(prop.get("development"))),
|
||||
"Listing features": key_features,
|
||||
"first_visible_date": prop.get("firstVisibleDate", ""),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Zoopla scraping via FlareSolverr (no browser/VNC needed).
|
||||
|
||||
FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which
|
||||
still contains the React Server Components flight stream — so the existing pure
|
||||
still contains the React Server Components flight stream, so the existing pure
|
||||
parsers work unchanged:
|
||||
- the search page yields the outcode's listing detail URLs, and
|
||||
- each detail page's flight stream carries the property's location object
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type { TFunction } from 'i18next';
|
|||
import { useTranslation } from 'react-i18next';
|
||||
import { cellToLatLng, polygonToCells } from 'h3-js';
|
||||
import PriceHistoryChart from '../map/PriceHistoryChart';
|
||||
import { MapErrorBoundary } from '../map/MapErrorBoundary';
|
||||
import StackedBarChart from '../map/StackedBarChart';
|
||||
import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions';
|
||||
import { DualHistogram } from '../map/DualHistogram';
|
||||
|
|
@ -214,10 +215,13 @@ const SHOWCASE_MAP_START_VIEW: ViewState = {
|
|||
bearing: 0,
|
||||
};
|
||||
|
||||
// End the Match fly-in over Greater London so the "match" step lands on the same
|
||||
// place the Inspect (SW5 9AA) and Scout (SW5/SE22/N4) steps then drill into. The
|
||||
// single-search narrative used to break by zooming to Birmingham then showing London.
|
||||
const SHOWCASE_MAP_END_VIEW: ViewState = {
|
||||
longitude: -1.89,
|
||||
latitude: 52.49,
|
||||
zoom: 8.72,
|
||||
longitude: -0.12,
|
||||
latitude: 51.51,
|
||||
zoom: 9.05,
|
||||
pitch: 0,
|
||||
bearing: 0,
|
||||
};
|
||||
|
|
@ -260,7 +264,12 @@ function buildShowcaseMapData(): HexagonData[] {
|
|||
}
|
||||
|
||||
const SHOWCASE_MAP_DATA = buildShowcaseMapData();
|
||||
const SHOWCASE_MAP_TOTAL_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => sum + item.count, 0);
|
||||
// Count only the hexes around Greater London so the Match card's figure matches the
|
||||
// area it has zoomed to, instead of mislabelling the England-wide total as one city.
|
||||
const SHOWCASE_LONDON_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => {
|
||||
const nearLondon = Math.abs(item.lat - 51.5072) < 0.55 && Math.abs(item.lon + 0.1276) < 0.85;
|
||||
return nearLondon ? sum + item.count : sum;
|
||||
}, 0);
|
||||
const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = [];
|
||||
const EMPTY_SHOWCASE_POIS: POI[] = [];
|
||||
|
||||
|
|
@ -601,6 +610,10 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) {
|
|||
return (
|
||||
<div className="pointer-events-none relative h-full overflow-hidden bg-warm-100 dark:bg-navy-950/50">
|
||||
{shouldRenderMap && (
|
||||
// The full deck.gl/maplibre map sits above the fold; contain any WebGL
|
||||
// context-loss crash to this pane (and auto-recover) instead of letting it
|
||||
// bubble to the top-level boundary and blank the hero.
|
||||
<MapErrorBoundary>
|
||||
<Suspense fallback={<div className="h-full bg-navy-950/40" aria-hidden="true" />}>
|
||||
<ProductMap
|
||||
data={SHOWCASE_MAP_DATA}
|
||||
|
|
@ -623,15 +636,16 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) {
|
|||
screenshotMode
|
||||
hideLegend
|
||||
densityLabel={t('home.showcaseMatchingHomesLabel')}
|
||||
totalCount={SHOWCASE_MAP_TOTAL_COUNT}
|
||||
totalCount={SHOWCASE_LONDON_COUNT}
|
||||
/>
|
||||
</Suspense>
|
||||
</MapErrorBoundary>
|
||||
)}
|
||||
<div className="pointer-events-none absolute bottom-4 left-4 max-w-[16rem] rounded-md border border-white/15 bg-navy-950/95 px-4 py-3 text-white shadow-2xl shadow-navy-950/35">
|
||||
<div className="text-sm font-black leading-none">Birmingham</div>
|
||||
<div className="text-sm font-black leading-none">{t('home.showcaseStep2Region')}</div>
|
||||
<div className="mt-1 text-xs font-bold leading-tight text-warm-200">
|
||||
{t('home.showcaseMatchingHomes', {
|
||||
value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(),
|
||||
value: SHOWCASE_LONDON_COUNT.toLocaleString(),
|
||||
})}
|
||||
</div>
|
||||
<div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400">
|
||||
|
|
@ -789,19 +803,22 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
|
|||
postcode: 'SW5 9AA',
|
||||
score: '94%',
|
||||
commute: t('home.showcaseMinutes', { count: 23 }),
|
||||
price: '£492k',
|
||||
perSqm: '£8,200',
|
||||
delta: '−16%',
|
||||
},
|
||||
{
|
||||
postcode: 'SE22 8EF',
|
||||
score: '91%',
|
||||
commute: t('home.showcaseMinutes', { count: 28 }),
|
||||
price: '£518k',
|
||||
perSqm: '£5,900',
|
||||
delta: '−24%',
|
||||
},
|
||||
{
|
||||
postcode: 'N4 2AB',
|
||||
score: '88%',
|
||||
commute: t('home.showcaseMinutes', { count: 31 }),
|
||||
price: '£476k',
|
||||
perSqm: '£6,400',
|
||||
delta: '−21%',
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -934,7 +951,12 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
|
|||
<div className="truncate border-r border-warm-200 px-2 py-1.5 dark:border-navy-700 sm:px-3 sm:py-2.5">
|
||||
{row.commute}
|
||||
</div>
|
||||
<div className="truncate px-2 py-1.5 font-black sm:px-3 sm:py-2.5">{row.price}</div>
|
||||
<div className="px-2 py-1.5 sm:px-3 sm:py-2.5">
|
||||
<div className="truncate font-black tabular-nums">{row.perSqm}</div>
|
||||
<div className="text-[10px] font-bold tabular-nums text-emerald-600 dark:text-emerald-300">
|
||||
{row.delta}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
|
@ -1031,6 +1053,7 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
|
|||
const [isStagePaused, setIsStagePaused] = useState(false);
|
||||
const [hasStarted, setHasStarted] = useState(false);
|
||||
const [canPauseOnHover, setCanPauseOnHover] = useState(false);
|
||||
const [prefersReducedMotion, setPrefersReducedMotion] = useState(false);
|
||||
const showcaseRef = useRef<HTMLDivElement | null>(null);
|
||||
const inspectUserScrolledRef = useRef(false);
|
||||
|
||||
|
|
@ -1101,6 +1124,27 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
|
|||
return () => mediaQuery.removeEventListener('change', updateCanPause);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof window.matchMedia !== 'function') return;
|
||||
const mediaQuery = window.matchMedia('(prefers-reduced-motion: reduce)');
|
||||
const update = () => setPrefersReducedMotion(mediaQuery.matches);
|
||||
update();
|
||||
mediaQuery.addEventListener('change', update);
|
||||
return () => mediaQuery.removeEventListener('change', update);
|
||||
}, []);
|
||||
|
||||
// Under prefers-reduced-motion the progress-bar animation is disabled in CSS, so
|
||||
// its onAnimationEnd (which normally advances the carousel) never fires and the
|
||||
// demo would freeze on step 1. Drive the advance from a timer in that case.
|
||||
useEffect(() => {
|
||||
if (!prefersReducedMotion || !isProgressRunning) return;
|
||||
const timer = window.setTimeout(
|
||||
() => setActiveStep((step) => (step + 1) % SHOWCASE_STEP_COUNT),
|
||||
activeStepIntervalMs
|
||||
);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [prefersReducedMotion, isProgressRunning, activeStep, activeStepIntervalMs]);
|
||||
|
||||
const pauseForHover = () => {
|
||||
if (canPauseOnHover) setIsStagePaused(true);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ export interface HistoricalPrice {
|
|||
year: number;
|
||||
month: number;
|
||||
price: number;
|
||||
is_new: boolean;
|
||||
}
|
||||
|
||||
export interface Property {
|
||||
|
|
@ -386,7 +387,7 @@ export interface HexagonStatsResponse {
|
|||
* sector, shown alongside the national average for each crime metric. */
|
||||
crime_area_averages?: CrimeAreaAverage[];
|
||||
/** Total individual crime records (last 7 years) across the selection's
|
||||
* postcodes — the count behind the "individual crimes" list. */
|
||||
* postcodes, the count behind the "individual crimes" list. */
|
||||
crime_total_records?: number;
|
||||
central_postcode?: string;
|
||||
/** Total usual residents (ONS Census 2021) across the postcodes in this
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Downloads five-year age band counts (TS007A) from the NOMIS API, then computes
|
||||
the median age per LSOA using linear interpolation within the median class.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS007A dataset, NM_2020_1)
|
||||
Source: NOMIS (ONS Census 2021, TS007A dataset, NM_2020_1)
|
||||
License: Open Government Licence v3.0
|
||||
"""
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv
|
|||
BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE"
|
||||
|
||||
# Five-year age bands in order, with lower bounds for interpolation.
|
||||
# The last band (85+) is open-ended — we treat it as 85-89 for median purposes.
|
||||
# The last band (85+) is open-ended. We treat it as 85-89 for median purposes.
|
||||
AGE_BANDS = [
|
||||
(0, 5), # Aged 0 to 4 years
|
||||
(5, 5), # Aged 5 to 9 years
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU"
|
|||
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
|
||||
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
|
||||
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
|
||||
# to its own "DLR station" category — restoring DLR to the train/tube station
|
||||
# to its own "DLR station" category, restoring DLR to the train/tube station
|
||||
# list and giving it the DLR roundel rather than a generic tram icon.
|
||||
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
|
||||
|
||||
|
|
@ -337,7 +337,7 @@ class StationAccumulator:
|
|||
# A single node is never both (ZZLU vs ZZDL), but a co-located
|
||||
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
|
||||
# and DLR halves into one group carrying both flags; Tube is checked
|
||||
# first so these resolve to "Tube station" — their primary identity —
|
||||
# first so these resolve to "Tube station" (their primary identity),
|
||||
# leaving "DLR station" for the DLR-only stops the fix targets.
|
||||
if self.category == TRAM_METRO_CATEGORY and self.is_lu:
|
||||
return TUBE_STATION_CATEGORY
|
||||
|
|
|
|||
|
|
@ -85,24 +85,24 @@ RESOLUTION = NATIVE_RESOLUTION
|
|||
|
||||
# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
|
||||
# otherwise inside the raster means "modelled below the lowest reporting band",
|
||||
# i.e. genuinely quiet — see noise_overlay_tiles.py:167.
|
||||
# i.e. genuinely quiet (see noise_overlay_tiles.py:167).
|
||||
NOISE_NODATA_SENTINEL = np.float32(-96.0)
|
||||
|
||||
# Lowest modelled Defra Lden reporting band (dB). Verified against the actual
|
||||
# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
|
||||
# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor
|
||||
# (0, 40). Below the band, cells are encoded as 0.0 (genuinely quiet). We floor
|
||||
# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
|
||||
# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
|
||||
# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
|
||||
# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
|
||||
# NOISE_COLOR_STOPS[0]) — a rendering threshold, not the data's reporting floor.
|
||||
# NOISE_COLOR_STOPS[0]), a rendering threshold, not the data's reporting floor.
|
||||
NOISE_QUIET_FLOOR_DB = np.float32(40.0)
|
||||
|
||||
# Sample noise at the postcode representative point itself (no neighbourhood
|
||||
# window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha
|
||||
# of every postcode; because Defra road contours hug every modelled road and
|
||||
# representative points sit on/near streets, that inflated postcode noise by
|
||||
# roughly +9 dB (log scale) — making ~94% of England read >=55 dB Lden and
|
||||
# roughly +9 dB (log scale), making ~94% of England read >=55 dB Lden and
|
||||
# collapsing the metric's discrimination at the quiet end. Radius 0 ->
|
||||
# filter_size 1 -> the maximum_filter is skipped and each postcode reads the
|
||||
# 10m cell it actually sits in.
|
||||
|
|
@ -110,8 +110,8 @@ POSTCODE_NOISE_RADIUS_M = 0
|
|||
|
||||
# Adjacent download tiles overlap by the sampling radius so every postcode's
|
||||
# sampling footprint is fully contained in at least one tile. With point
|
||||
# sampling (radius 0) this is 0 — a representative point falls inside exactly
|
||||
# one tile — but the relationship is kept so any future non-zero radius keeps
|
||||
# sampling (radius 0) this is 0 (a representative point falls inside exactly
|
||||
# one tile), but the relationship is kept so any future non-zero radius keeps
|
||||
# its window seam-safe.
|
||||
TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
|
||||
|
||||
|
|
@ -273,7 +273,7 @@ def _download_tile(
|
|||
NoGeoTiffError,
|
||||
httpx.HTTPStatusError,
|
||||
# TransportError is the superset of TimeoutException, ConnectError,
|
||||
# ReadError and ProtocolError — including RemoteProtocolError, raised
|
||||
# ReadError and ProtocolError, including RemoteProtocolError, raised
|
||||
# when the WCS server closes the connection mid-stream ("incomplete
|
||||
# chunked read"). All are transient; retry/split rather than letting
|
||||
# one flaky tile crash the whole raster download.
|
||||
|
|
@ -446,7 +446,7 @@ def sample_noise_at_postcodes(
|
|||
# Defra rasters encode TRUE nodata as the -96.0 sentinel (and
|
||||
# occasionally non-finite / dataset.nodata); genuinely quiet ground
|
||||
# below the model's lowest reporting band is encoded as 0.0. Only
|
||||
# the former is "we don't know" — the latter is a real "we know it's
|
||||
# the former is "we don't know". The latter is a real "we know it's
|
||||
# quiet" reading and must not collapse to null. So treat ONLY true
|
||||
# nodata as -inf (it never wins a max and never counts as coverage),
|
||||
# and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
|
||||
|
|
@ -555,7 +555,7 @@ def main() -> None:
|
|||
|
||||
if not tile_paths:
|
||||
print(
|
||||
f"[{label}] WARNING: No tiles downloaded — column will be all null"
|
||||
f"[{label}] WARNING: No tiles downloaded; column will be all null"
|
||||
)
|
||||
series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ site id and the site's polygon centroid. Sites without access points fall
|
|||
back to polygon centroids.
|
||||
|
||||
Using access points rather than polygon centroids gives much more accurate
|
||||
distance calculations — a property next to Hyde Park won't show 400m just
|
||||
distance calculations: a property next to Hyde Park won't show 400m just
|
||||
because the centroid is in the middle of the park. The site id / centroid
|
||||
columns let downstream consumers (poi_proximity) collapse the frame back to
|
||||
one row per SITE for counting, so a park with 30 gates counts as one park.
|
||||
|
|
@ -190,7 +190,7 @@ def download_greenspace(output: Path) -> None:
|
|||
print(f"Reading {site_shps[0].name} for function types...")
|
||||
site_funcs = _read_site_functions(site_shps[0])
|
||||
|
||||
# Step 2: Read access points (primary — park entrances)
|
||||
# Step 2: Read access points (primary: park entrances)
|
||||
print(f"Reading {access_shps[0].name}...")
|
||||
ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points(
|
||||
access_shps[0], site_funcs
|
||||
|
|
|
|||
|
|
@ -930,7 +930,7 @@ def main() -> None:
|
|||
df.write_parquet(args.output)
|
||||
print(f"Saved to {args.output}")
|
||||
else:
|
||||
print("No places found — skipping output")
|
||||
print("No places found, skipping output")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ def _data_rows(df: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
The preamble length varies (title, optional "This worksheet contains..."
|
||||
note, then the header row starting with "Time period"), so locate the
|
||||
header by content instead of counting rows — a fixed slice leaves the
|
||||
header by content instead of counting rows: a fixed slice leaves the
|
||||
header in the data whenever ONS adds or removes a note line.
|
||||
"""
|
||||
header_marker = (
|
||||
|
|
@ -78,7 +78,7 @@ def _latest_rents_long(df: pl.DataFrame) -> pl.DataFrame:
|
|||
print(f"LAs in latest month: {df.height}")
|
||||
|
||||
# Melt to long format: one row per area x bedroom count.
|
||||
# PIPR has no Studio category — one-bed rent used as proxy for bedrooms=0.
|
||||
# PIPR has no Studio category: one-bed rent used as proxy for bedrooms=0.
|
||||
frames = []
|
||||
for col, bedrooms in [
|
||||
("rent_1bed", 0), # Studio (proxy)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9)
|
||||
from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf
|
||||
categories into our 3 output buckets — Owner occupied / Social rent /
|
||||
Private rent — emitting one row per LSOA with the percentage of households in
|
||||
categories into our 3 output buckets (Owner occupied / Social rent /
|
||||
Private rent), emitting one row per LSOA with the percentage of households in
|
||||
each. The three buckets sum to 100%, so downstream they render as a
|
||||
composition/ratio (like the ethnicity and qualifications stacked bars) AND each
|
||||
percentage is independently filterable.
|
||||
|
|
@ -19,7 +19,7 @@ NOTE this table counts HOUSEHOLDS (not usual residents). The join key downstream
|
|||
(merge.py) is `lsoa21`, the same key used for ethnicity, qualifications,
|
||||
median age, and IoD.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS054 dataset, NM_2072_1)
|
||||
Source: NOMIS (ONS Census 2021, TS054 dataset, NM_2072_1)
|
||||
License: Open Government Licence v3.0
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ def test_ethnicity_routes_chinese_to_east_and_other_asian_to_se():
|
|||
|
||||
|
||||
def test_ethnicity_percentages_independent_per_lsoa():
|
||||
"""Two LSOAs get independent profiles — the LSOA granularity is the point."""
|
||||
"""Two LSOAs get independent profiles: the LSOA granularity is the point."""
|
||||
df = pl.concat(
|
||||
[
|
||||
pl.DataFrame(
|
||||
|
|
|
|||
|
|
@ -281,7 +281,7 @@ def test_sample_noise_preserves_genuine_reading_above_quiet_floor(
|
|||
|
||||
# The lowest Defra reporting band is 40.0 dB; genuine readings populate
|
||||
# [40, ~80]. A genuine in-coverage reading at or just above the floor must be
|
||||
# PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A
|
||||
# PRESERVED, not clamped UP to the floor. Only true-quiet 0.0 is floored. A
|
||||
# quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
|
||||
# dB readings; this pins that they survive unchanged.
|
||||
floor = float(noise.NOISE_QUIET_FLOOR_DB)
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ ENGLAND_PBF_URL = (
|
|||
"https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf"
|
||||
)
|
||||
|
||||
# Bus Open Data Service — pre-converted GTFS covering all England bus/tram/ferry
|
||||
# Bus Open Data Service: pre-converted GTFS covering all England bus/tram/ferry
|
||||
BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
|
||||
|
||||
# National Rail Open Data API
|
||||
|
|
@ -597,7 +597,7 @@ def validate_gtfs_feed(
|
|||
fail("has neither calendar.txt nor calendar_dates.txt")
|
||||
if not _calendar_active_in_window(z, names, window_start, window_end):
|
||||
fail(
|
||||
f"no service active between {window_start} and {window_end} — "
|
||||
f"no service active between {window_start} and {window_end}: "
|
||||
"the feed's calendars are stale/expired and it would contribute "
|
||||
"zero service to routing"
|
||||
)
|
||||
|
|
@ -697,7 +697,7 @@ def _iter_stop_time_trips(lines, trip_id_idx: int):
|
|||
dtd2mysql currently writes rows grouped by trip and ordered by
|
||||
stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a
|
||||
trip_id reappearing later raises instead of silently scrambling trips);
|
||||
within-trip order is NOT assumed — callers sort each group by its original
|
||||
within-trip order is NOT assumed: callers sort each group by its original
|
||||
stop_sequence.
|
||||
"""
|
||||
current_trip: str | None = None
|
||||
|
|
@ -1175,7 +1175,7 @@ def main() -> None:
|
|||
cif = download_national_rail_cif(raw_dir)
|
||||
if cif is None:
|
||||
raise RuntimeError(
|
||||
"National Rail timetable was not downloaded — set "
|
||||
"National Rail timetable was not downloaded: set "
|
||||
"NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at "
|
||||
"https://opendata.nationalrail.co.uk/). National Rail heavy rail is "
|
||||
"required; without it the transit network models every train journey "
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
"""Robust GEOS overlay helpers.
|
||||
|
||||
Overlay operations (union, difference, intersection) can raise a
|
||||
``GEOSException`` — most often ``TopologyException: side location conflict``,
|
||||
``Ring edge missing``, or ``found non-noded intersection`` — on geometries that
|
||||
``GEOSException``, most often ``TopologyException: side location conflict``,
|
||||
``Ring edge missing``, or ``found non-noded intersection``, on geometries that
|
||||
contain near-coincident or near-degenerate edges, or that are individually
|
||||
invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG
|
||||
engine, handed a grid size, nodes every edge onto that grid and finishes where
|
||||
|
|
@ -14,8 +14,8 @@ learned the hard way from a crash:
|
|||
1. **Never precision-reduce with the default mode.** ``set_precision``'s default
|
||||
``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that
|
||||
re-raises the very ``side location conflict`` we are trying to escape. We push
|
||||
the grid into the overlay via the ``grid_size`` argument instead — where
|
||||
OverlayNG nodes robustly — and only ever call ``set_precision`` in
|
||||
the grid into the overlay via the ``grid_size`` argument instead (where
|
||||
OverlayNG nodes robustly) and only ever call ``set_precision`` in
|
||||
``pointwise`` mode (pure coordinate rounding, which cannot raise).
|
||||
2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties,
|
||||
pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape.
|
||||
|
|
@ -40,7 +40,7 @@ from shapely import GEOSException, make_valid, set_precision
|
|||
from shapely.geometry import Polygon
|
||||
from shapely.ops import unary_union
|
||||
|
||||
# 0.1 mm in metres — well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
|
||||
# 0.1 mm in metres: well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
|
||||
_SNAP_GRID = 1e-4
|
||||
|
||||
_EMPTY = Polygon()
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ def _is_pointlike(geom_bng) -> bool:
|
|||
def _rescue_footprint(geom_bng) -> dict | None:
|
||||
"""Fatten a degenerate BNG geometry into a representable footprint and snap.
|
||||
|
||||
A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon — the
|
||||
A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon, the
|
||||
signature of a tower-block postcode whose UPRNs all share one coordinate)
|
||||
gets a building-scale buffer so it is not reduced to an invisible sub-metre
|
||||
dot; thin slivers that still carry length keep the minimal buffer.
|
||||
|
|
@ -263,7 +263,7 @@ def merge_fragments(
|
|||
# Close tiny gaps between adjacent OA boundary edges (float mismatches).
|
||||
# The closing can erode a tiny MultiPolygon (e.g. a postcode with only a
|
||||
# sliver fragment) to nothing, which would leave the postcode with no
|
||||
# geometry at all — keep the un-closed shape if that happens.
|
||||
# geometry at all. Keep the un-closed shape if that happens.
|
||||
if combined.geom_type == "MultiPolygon":
|
||||
closed = combined.buffer(5.0).buffer(-5.0)
|
||||
if not closed.is_valid:
|
||||
|
|
@ -308,7 +308,7 @@ def _polygonal(geom):
|
|||
return None
|
||||
# Both callers run on WGS84-degree output geometry, so the robustness
|
||||
# fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre
|
||||
# default — a coarse metre grid would obliterate a degree-scale shape.
|
||||
# default. A coarse metre grid would obliterate a degree-scale shape.
|
||||
merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG)
|
||||
return merged if not merged.is_empty else None
|
||||
return None
|
||||
|
|
@ -324,7 +324,7 @@ def _resolve_overlaps(
|
|||
containment (a postcode fully enclosed by another). Each postcode is trimmed
|
||||
by the union of its higher-priority overlapping neighbours, where **priority =
|
||||
ascending area**: a smaller postcode wins contested ground. That single rule
|
||||
handles both cases correctly — an enclosed postcode is always smaller than its
|
||||
handles both cases correctly: an enclosed postcode is always smaller than its
|
||||
container, so it keeps its area while the container gets a hole (a `overlaps`
|
||||
query alone would miss containment entirely). Run last, on the final output
|
||||
geometries, so nothing re-introduces overlap afterwards. A postcode that would
|
||||
|
|
@ -348,7 +348,7 @@ def _resolve_overlaps(
|
|||
arr = np.array(geoms, dtype=object)
|
||||
pairs: set[tuple[int, int]] = set()
|
||||
# "overlaps" gives partial overlaps; "contains" gives containment (which
|
||||
# "overlaps" excludes) — together they cover every 2-D overlap without the
|
||||
# "overlaps" excludes). Together they cover every 2-D overlap without the
|
||||
# edge-touch explosion a plain "intersects" query would add.
|
||||
for predicate in ("overlaps", "contains"):
|
||||
qsrc, qtgt = tree.query(arr, predicate=predicate)
|
||||
|
|
@ -577,7 +577,7 @@ def _grid_footprint(geom):
|
|||
pass can shave a small (e.g. co-located, non-geographic) postcode down to a
|
||||
sub-grid sliver that disappears when snapped to output precision. Rather than
|
||||
drop it, place a minimal valid footprint at its location. The tiny overlap
|
||||
this re-creates with the neighbour that trimmed it is harmless — the output
|
||||
this re-creates with the neighbour that trimmed it is harmless: the output
|
||||
partition is best-effort, a missing boundary is a hard validation failure.
|
||||
"""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from .voronoi import compute_voronoi_regions
|
|||
MIN_GEOM_AREA = 0.01
|
||||
|
||||
# Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a
|
||||
# crowded multi-postcode OA — its Voronoi cell ∩ remaining collapses below
|
||||
# crowded multi-postcode OA: its Voronoi cell ∩ remaining collapses below
|
||||
# MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a
|
||||
# co-located postcode. Every *active* postcode must keep a boundary
|
||||
# (validate_outputs is zero-tolerance), so it gets a small disc at its true seed
|
||||
|
|
@ -77,7 +77,7 @@ def process_oa(
|
|||
fragments.append((pc, merged))
|
||||
|
||||
# Every postcode with a UPRN seed in this OA must keep at least a minimal
|
||||
# footprint — in a dense OA (a block of flats with hundreds of distinct
|
||||
# footprint: in a dense OA (a block of flats with hundreds of distinct
|
||||
# postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or
|
||||
# be fully absorbed by a co-located postcode's INSPIRE parcel, producing no
|
||||
# fragment, and an active postcode must never be dropped.
|
||||
|
|
@ -142,7 +142,7 @@ def _claim_inspire_parcels(
|
|||
# UPRNs from a single postcode goes wholly to that postcode. A parcel shared
|
||||
# by several postcodes (a block of flats spanning postcodes, or overlapping
|
||||
# parcel data) is split between them via a sub-Voronoi over their own UPRNs
|
||||
# clipped to the parcel — so EVERY contained postcode keeps part of the
|
||||
# clipped to the parcel, so EVERY contained postcode keeps part of the
|
||||
# parcel. A bare majority vote would hand the whole parcel to one winner and
|
||||
# leave the losers' UPRNs trapped inside claimed land, dropping them from
|
||||
# both this claim and the `remaining` polygon handed to Voronoi downstream.
|
||||
|
|
@ -312,7 +312,7 @@ def _extract_polygonal(geom) -> Polygon | MultiPolygon | None:
|
|||
return polys[0]
|
||||
# Union (not bare MultiPolygon construction): make_valid can emit
|
||||
# overlapping polygonal parts, and a MultiPolygon of overlapping parts is
|
||||
# invalid — it double-counts area and makes the next `.difference()` raise
|
||||
# invalid: it double-counts area and makes the next `.difference()` raise
|
||||
# a TopologyException that aborts the OA (and, in parallel mode, the
|
||||
# worker). safe_union merges them into a valid geometry.
|
||||
merged = safe_union(polys)
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ def load_uprns(
|
|||
# Remap terminated postcodes to their nearest active successor. The
|
||||
# successor generally lives in a DIFFERENT OA (and at different grid
|
||||
# coordinates), so the remapped point must adopt the successor's
|
||||
# authoritative OA/coords — keeping the terminated postcode's original
|
||||
# authoritative OA/coords. Keeping the terminated postcode's original
|
||||
# OA would seed the successor into an OA it doesn't belong to, splitting
|
||||
# its boundary across OAs. Genuine (non-remapped) UPRN rows keep their
|
||||
# own OA, since a live postcode can legitimately span several OAs.
|
||||
|
|
@ -127,7 +127,7 @@ def load_uprns(
|
|||
uprns.sort("OA21CD").sink_parquet(tmp_path)
|
||||
release_memory()
|
||||
|
||||
# Read the sorted data — only one copy in memory (~2GB)
|
||||
# Read the sorted data: only one copy in memory (~2GB)
|
||||
df = pl.read_parquet(tmp_path)
|
||||
tmp_path.unlink()
|
||||
n = len(df)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Stratified by property type and postcode sector, with IRLS Huber regression,
|
|||
hierarchical shrinkage (sector → district → area → national → hedonic),
|
||||
and KD-tree spatial smoothing for sparse sectors.
|
||||
|
||||
Output: price_index.parquet — sector x type_group x year -> log_index
|
||||
Output: price_index.parquet (sector x type_group x year -> log_index)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
|
|||
# Natural join key from estimates back onto properties: postcode plus the
|
||||
# coalesced register/EPC address. This is unique and non-null on the deduped
|
||||
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
|
||||
# estimates 1:1 onto properties regardless of row order — estimates are computed
|
||||
# estimates 1:1 onto properties regardless of row order. Estimates are computed
|
||||
# from a separate price_inputs.parquet, so a positional key would not line up.
|
||||
JOIN_ADDRESS = "_join_address"
|
||||
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
|
||||
|
|
|
|||
|
|
@ -315,7 +315,7 @@ def test_run_tenure_history_tracks_rent_owner_transitions(tmp_path: Path):
|
|||
|
||||
def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
|
||||
# A property only ever observed as owner-occupied has no tenure change worth
|
||||
# surfacing — the timeline column is null (no events), not a noisy baseline.
|
||||
# surfacing: the timeline column is null (no events), not a noisy baseline.
|
||||
zip_path = tmp_path / "domestic-csv.zip"
|
||||
_write_epc_zip(
|
||||
zip_path,
|
||||
|
|
@ -339,6 +339,37 @@ def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
|
|||
assert df.get_column("tenure_history").to_list() == [None]
|
||||
|
||||
|
||||
def test_run_latest_tenure_status_reflects_most_recent_certificate(tmp_path: Path):
|
||||
# Two certificates for one dwelling: an older social-rented cert and a newer
|
||||
# owner-occupied one. The published latest_tenure_status must carry the
|
||||
# LATEST certificate's normalized tenure ("Owner-occupied"), while
|
||||
# was_council_house stays "Yes" because the dwelling was social at some
|
||||
# point. This also confirms latest_tenure_status reaches the epc_pp parquet.
|
||||
zip_path = tmp_path / "domestic-csv.zip"
|
||||
_write_epc_zip(
|
||||
zip_path,
|
||||
[
|
||||
_row(inspection_date="2016-04-01", tenure="Rented (social)"),
|
||||
_row(inspection_date="2024-04-01", tenure="owner-occupied"),
|
||||
],
|
||||
)
|
||||
|
||||
price_paid_path = tmp_path / "price-paid.parquet"
|
||||
_price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet(
|
||||
price_paid_path
|
||||
)
|
||||
|
||||
output_path = tmp_path / "epc-pp.parquet"
|
||||
_run(zip_path, price_paid_path, output_path, tmp_path)
|
||||
|
||||
df = pl.read_parquet(output_path)
|
||||
|
||||
assert df.height == 1
|
||||
assert df.select("latest_tenure_status", "was_council_house").to_dicts() == [
|
||||
{"latest_tenure_status": "Owner-occupied", "was_council_house": "Yes"}
|
||||
]
|
||||
|
||||
|
||||
def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path):
|
||||
# Two certificates for the same property. The cert with the garbled,
|
||||
# unparseable inspection_date must NOT be chosen as "latest": a string sort
|
||||
|
|
@ -565,7 +596,7 @@ def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: P
|
|||
price_paid_path = tmp_path / "price-paid.parquet"
|
||||
pl.DataFrame(
|
||||
{
|
||||
# 5_000 is below MIN_PRICE (10_000) — a nominal/junk transfer that
|
||||
# 5_000 is below MIN_PRICE (10_000), a nominal/junk transfer that
|
||||
# must still anchor the construction year but stay out of the price
|
||||
# aggregations.
|
||||
"price": [5_000, 300_000],
|
||||
|
|
@ -603,7 +634,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
|
|||
# lands AFTER its first Land Registry sale (1998). A dwelling cannot have
|
||||
# been built after it was first sold, so the published build year must be
|
||||
# capped at the first transfer year (1998), not the later band estimate.
|
||||
# It stays flagged as an estimate (approximate=1) — it is still EPC-derived.
|
||||
# It stays flagged as an estimate (approximate=1). It is still EPC-derived.
|
||||
zip_path = tmp_path / "domestic-csv.zip"
|
||||
_write_epc_zip(
|
||||
zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")]
|
||||
|
|
@ -627,7 +658,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
|
|||
|
||||
def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
|
||||
# The common case: the EPC band (1950-1966 -> 1958) predates the first
|
||||
# recorded sale (2020). The cap must NOT fire — the band estimate stands.
|
||||
# recorded sale (2020). The cap must NOT fire. The band estimate stands.
|
||||
zip_path = tmp_path / "domestic-csv.zip"
|
||||
_write_epc_zip(zip_path)
|
||||
|
||||
|
|
@ -649,7 +680,7 @@ def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
|
|||
def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
|
||||
# A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the
|
||||
# NEW floor (10k): it must now be RETAINED in the price aggregations. This
|
||||
# pins the 50k->10k change — it fails on the pre-fix 50k floor (where 30k was
|
||||
# pins the 50k->10k change: it fails on the pre-fix 50k floor (where 30k was
|
||||
# excluded, giving historical_prices length 1 / latest_price 250_000).
|
||||
zip_path = tmp_path / "domestic-csv.zip"
|
||||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
|
|
@ -748,8 +779,8 @@ def test_run_collapses_duplicate_transactions(tmp_path: Path):
|
|||
# The duplicated 250_000 sale collapses to one entry; two distinct sales.
|
||||
assert df.get_column("historical_prices").to_list() == [
|
||||
[
|
||||
{"year": 2020, "month": 2, "price": 200_000},
|
||||
{"year": 2024, "month": 2, "price": 250_000},
|
||||
{"year": 2020, "month": 2, "price": 200_000, "is_new": False},
|
||||
{"year": 2024, "month": 2, "price": 250_000, "is_new": False},
|
||||
]
|
||||
]
|
||||
assert df.get_column("latest_price").to_list() == [250_000]
|
||||
|
|
@ -777,7 +808,7 @@ def test_run_excludes_implausible_price_jump_but_keeps_property(tmp_path: Path):
|
|||
assert df.height == 1
|
||||
assert df.get_column("latest_price").to_list() == [140_000]
|
||||
assert df.get_column("historical_prices").to_list() == [
|
||||
[{"year": 2016, "month": 6, "price": 140_000}]
|
||||
[{"year": 2016, "month": 6, "price": 140_000, "is_new": False}]
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -593,7 +593,7 @@ def test_transform_grocery_dedup_drops_only_grocery_aspect(tmp_path):
|
|||
# The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at
|
||||
# (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name
|
||||
# is the same physical store, so its Convenience Store (Groceries) row is a
|
||||
# duplicate and must be dropped — but its NON-grocery aspect (a Post Office
|
||||
# duplicate and must be dropped, but its NON-grocery aspect (a Post Office
|
||||
# sharing the same OSM id) must survive. An independent shop away from the
|
||||
# GEOLYTIX point keeps its grocery row.
|
||||
raw = pl.DataFrame(
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ DROP_CATEGORIES = {
|
|||
"emergency/water_tank",
|
||||
"leisure/bleachers",
|
||||
"leisure/schoolyard",
|
||||
# Park "furniture" / incidental features — not parks; they massively
|
||||
# Park "furniture" / incidental features, not parks; they massively
|
||||
# inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k).
|
||||
"leisure/bandstand",
|
||||
"leisure/bird_hide",
|
||||
|
|
@ -222,7 +222,7 @@ DROP_CATEGORIES = {
|
|||
"public_transport/entrance",
|
||||
"public_transport/station",
|
||||
"public_transport/stop_position",
|
||||
# Education amenities — schools come from GIAS instead. OSM coverage for
|
||||
# Education amenities. Schools come from GIAS instead. OSM coverage for
|
||||
# tertiary education, tutoring, and childcare is too noisy/incomplete to be
|
||||
# useful on a property-search map.
|
||||
"amenity/school",
|
||||
|
|
@ -398,7 +398,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
|||
"tourism/theme_park",
|
||||
# bicycle_rental/boat_rental/marina/slipway used to live here and
|
||||
# made up ~46% of the bucket (cycle-hire docks, boat ramps); they
|
||||
# are infrastructure, not entertainment venues — see DROP_CATEGORIES.
|
||||
# are infrastructure, not entertainment venues. See DROP_CATEGORIES.
|
||||
"leisure/hackerspace",
|
||||
"leisure/yes",
|
||||
],
|
||||
|
|
@ -722,7 +722,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
|||
[
|
||||
"leisure/fitness_centre",
|
||||
# leisure/fitness_station (outdoor pull-up bars / trim-trail
|
||||
# apparatus, ~2.5k) is not a gym — see DROP_CATEGORIES.
|
||||
# apparatus, ~2.5k) is not a gym. See DROP_CATEGORIES.
|
||||
"amenity/dojo",
|
||||
"amenity/dancing_school",
|
||||
],
|
||||
|
|
@ -849,8 +849,8 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
|||
"healthcare/pharmacy",
|
||||
"shop/chemist",
|
||||
# healthcare/alternative, shop/herbalist and shop/health (homeopaths,
|
||||
# herbalists, generic "health" shops) are not dispensing pharmacies
|
||||
# — see DROP_CATEGORIES.
|
||||
# herbalists, generic "health" shops) are not dispensing pharmacies.
|
||||
# See DROP_CATEGORIES.
|
||||
],
|
||||
),
|
||||
# "Hospital & Clinic" used to be one bucket; an actual hospital and a small
|
||||
|
|
@ -878,7 +878,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
|||
"healthcare/laboratory",
|
||||
"healthcare/rehabilitation",
|
||||
"healthcare/vaccination_centre",
|
||||
# healthcare/yes (untyped junk rows) is dropped — see DROP_CATEGORIES.
|
||||
# healthcare/yes (untyped junk rows) is dropped. See DROP_CATEGORIES.
|
||||
],
|
||||
),
|
||||
(
|
||||
|
|
@ -950,7 +950,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
|||
[
|
||||
"tourism/gallery",
|
||||
# tourism/artwork (statues, murals, village signs) was 93% of this
|
||||
# bucket and is not a visitable gallery — see DROP_CATEGORIES.
|
||||
# bucket and is not a visitable gallery. See DROP_CATEGORIES.
|
||||
],
|
||||
),
|
||||
(
|
||||
|
|
@ -1420,7 +1420,7 @@ def _school_icon_category_expr() -> pl.Expr:
|
|||
# GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed
|
||||
# primary") so we normalise before matching.
|
||||
phase = pl.col("phase").str.to_lowercase()
|
||||
# gias._format_age_range emits three shapes: "<low>–<high>" (em-dash),
|
||||
# gias._format_age_range emits three shapes: "<low>–<high>" (en-dash),
|
||||
# "up to <high>" (high-only) and "<low>+" (low-only). Extract the leading
|
||||
# integer as low and the trailing integer as high, then suppress the wrong
|
||||
# end for the one-sided shapes so they don't collapse to a single bound.
|
||||
|
|
@ -1477,7 +1477,7 @@ def _load_ofsted_ratings(ofsted_path: Path) -> pl.LazyFrame:
|
|||
the conventional Ofsted labels; when there is no usable graded result
|
||||
(null/"Not judged", e.g. schools last seen under the post-2024 ungraded
|
||||
report-card framework) we fall back to "Ungraded inspection overall outcome"
|
||||
so genuinely good/outstanding schools aren't dropped — mirroring
|
||||
so genuinely good/outstanding schools aren't dropped, mirroring
|
||||
school_catchments.classify_good_plus_schools. Remaining nulls drop out."""
|
||||
grade_col = pl.col("Latest OEIF overall effectiveness")
|
||||
# See school_catchments: the ungraded outcome carries "School remains Good"/
|
||||
|
|
@ -1682,7 +1682,7 @@ def osm_groceries_colocated_with_geolytix(
|
|||
|
||||
An OSM Groceries row is a duplicate when a GEOLYTIX point lies within
|
||||
``radius_m`` metres AND that point's brand tokens (its ``category``, e.g.
|
||||
"Tesco", "Co-op", "M&S") are all present in the OSM row's name — i.e. the
|
||||
"Tesco", "Co-op", "M&S") are all present in the OSM row's name, i.e. the
|
||||
same physical branded store. Short brands like "M&S" match via
|
||||
_GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept.
|
||||
|
||||
|
|
@ -1703,7 +1703,7 @@ def osm_groceries_colocated_with_geolytix(
|
|||
osm_ids = osm_groceries["id"].to_list()
|
||||
osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()]
|
||||
|
||||
# Equirectangular projection to metres around the shared mean latitude — at
|
||||
# Equirectangular projection to metres around the shared mean latitude. At
|
||||
# England's scale this is accurate to well under the dedup radius.
|
||||
mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat])))
|
||||
cos_lat = float(np.cos(np.radians(mean_lat)))
|
||||
|
|
@ -1860,7 +1860,7 @@ def transform(
|
|||
)
|
||||
# Scope the drop to the Groceries group: a single OSM object can also
|
||||
# carry a non-grocery aspect (e.g. a convenience store that is also a
|
||||
# Post Office), which must survive — only its duplicate grocery row goes.
|
||||
# Post Office), which must survive. Only its duplicate grocery row goes.
|
||||
lf = lf.filter(
|
||||
~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids))
|
||||
)
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ POSTCODE_DENSITY_PERCENTILE_COL = "Tree canopy density percentile within {radius
|
|||
POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)"
|
||||
POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)"
|
||||
|
||||
# National Forest Inventory (NFI) woodland — the geometric complement of TOW.
|
||||
# National Forest Inventory (NFI) woodland: the geometric complement of TOW.
|
||||
# NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700.
|
||||
# Field names are from the NFI Woodland England 2022 release; re-check on bumps.
|
||||
NFI_CATEGORY_COL = "CATEGORY"
|
||||
|
|
@ -263,7 +263,7 @@ def _postcode_buffers(
|
|||
return circles, shapely.STRtree(circles)
|
||||
|
||||
|
||||
# 0.1 mm in the BNG working CRS (EPSG:27700) — far below survey resolution; the
|
||||
# 0.1 mm in the BNG working CRS (EPSG:27700), far below survey resolution; the
|
||||
# same grid the postcode_boundaries overlay uses.
|
||||
_OVERLAY_GRID_M = 1e-4
|
||||
|
||||
|
|
@ -274,13 +274,13 @@ def _robust_intersection_area(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
|||
External Forest Research TOW/NFI polygons are occasionally invalid
|
||||
(self-intersections), and a single bad polygon makes the batched
|
||||
``shapely.intersection`` raise ``TopologyException: side location conflict``,
|
||||
aborting the whole run. The fast path is the raw batched overlay — unchanged,
|
||||
full-speed, when the data is clean — and only a failure triggers repair.
|
||||
aborting the whole run. The fast path is the raw batched overlay (unchanged,
|
||||
full-speed, when the data is clean), and only a failure triggers repair.
|
||||
|
||||
The repair deliberately uses a *plain* overlay rather than the fixed-precision
|
||||
(``grid_size``) one: ``make_valid`` can emit a mixed-dimension
|
||||
``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG
|
||||
rejects with ``Overlay input is mixed-dimension`` — whereas a plain overlay
|
||||
rejects with ``Overlay input is mixed-dimension``, whereas a plain overlay
|
||||
accepts it, and its non-polygonal debris has zero area and is dropped by the
|
||||
``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate
|
||||
snap (which never raises) collapses the near-coincident edges behind any
|
||||
|
|
@ -496,7 +496,7 @@ def _finalize_metrics(
|
|||
if over_count:
|
||||
print(
|
||||
f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were "
|
||||
"capped — indicates overlapping TOW/NFI canopy within the buffer"
|
||||
"capped, indicating overlapping TOW/NFI canopy within the buffer"
|
||||
)
|
||||
|
||||
mean_height = np.divide(
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ def test_custom_radius(pois):
|
|||
}
|
||||
)
|
||||
|
||||
# 0.01 km = 10m — only the POI at the exact same location should match
|
||||
# 0.01 km = 10m: only the POI at the exact same location should match
|
||||
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01)
|
||||
# The Restaurant at (51.5074, -0.1278) is at distance 0
|
||||
assert result["restaurants_0km"][0] >= 1
|
||||
|
|
@ -135,9 +135,9 @@ def test_min_distance_finds_nearest(postcodes, pois):
|
|||
assert len(result) == 2
|
||||
|
||||
ec1a = result.filter(pl.col("postcode") == "EC1A 1BB")
|
||||
# Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278) — very close
|
||||
# Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278), very close
|
||||
assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m
|
||||
# Restaurant is co-located — distance ~0
|
||||
# Restaurant is co-located: distance ~0
|
||||
assert ec1a["restaurants_nearest_km"][0] < 0.01
|
||||
|
||||
# Far-away postcode should still get the global nearest distance.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue