.
This commit is contained in:
parent
909e241907
commit
1ee796b282
29 changed files with 250 additions and 126 deletions
|
|
@ -14,7 +14,7 @@ def _prop(pid, pin=None):
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _needs_detail_fetch — accurate-pin skip
|
# _needs_detail_fetch: accurate-pin skip
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -32,7 +32,7 @@ def test_needs_detail_fetch_disabled_always_fetches(monkeypatch):
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _prime_detail_postcodes — worklist selection + concurrent fetch
|
# _prime_detail_postcodes: worklist selection + concurrent fetch
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -97,7 +97,7 @@ def test_prime_is_a_noop_when_disabled_or_cap_zero(monkeypatch):
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _paginate — end-to-end (network stubbed): accurate pins fall back to
|
# _paginate end-to-end (network stubbed): accurate pins fall back to
|
||||||
# coordinates, approximate pins use the detail postcode.
|
# coordinates, approximate pins use the detail postcode.
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import zoopla
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _run_sources — Zoopla inline, others in threads, failures isolated
|
# _run_sources: Zoopla inline, others in threads, failures isolated
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -85,13 +85,13 @@ def test_seed_and_save_detail_caches_round_trip(tmp_path):
|
||||||
|
|
||||||
def test_seed_detail_caches_tolerates_missing_files(tmp_path):
|
def test_seed_detail_caches_tolerates_missing_files(tmp_path):
|
||||||
rightmove._detail_postcode_cache.clear()
|
rightmove._detail_postcode_cache.clear()
|
||||||
# No file written yet — seeding must not raise and must leave cache empty.
|
# No file written yet: seeding must not raise and must leave cache empty.
|
||||||
scraper._seed_detail_caches(["rightmove"], tmp_path)
|
scraper._seed_detail_caches(["rightmove"], tmp_path)
|
||||||
assert rightmove._detail_postcode_cache == {}
|
assert rightmove._detail_postcode_cache == {}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# run_scrape — full orchestration wiring (sources stubbed, no network)
|
# run_scrape: full orchestration wiring (sources stubbed, no network)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
from transform import (
|
from transform import (
|
||||||
|
build_listing_url,
|
||||||
build_register_address,
|
build_register_address,
|
||||||
clean_listing_address,
|
clean_listing_address,
|
||||||
extract_full_postcode,
|
extract_full_postcode,
|
||||||
|
|
@ -173,3 +174,33 @@ def test_rightmove_transform_without_detail_keeps_coordinate_logic() -> None:
|
||||||
assert result is not None
|
assert result is not None
|
||||||
assert result["Postcode"] == "SW9 7AA"
|
assert result["Postcode"] == "SW9 7AA"
|
||||||
assert result["Postcode source"] == "coordinates"
|
assert result["Postcode source"] == "coordinates"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_listing_url_stamps_channel_from_new_build_flag() -> None:
|
||||||
|
# Resale gets RES_BUY; new builds get RES_NEW.
|
||||||
|
assert build_listing_url("/properties/200", False) == (
|
||||||
|
"https://www.rightmove.co.uk/properties/200#/?channel=RES_BUY"
|
||||||
|
)
|
||||||
|
assert build_listing_url("/properties/200", True) == (
|
||||||
|
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
|
||||||
|
)
|
||||||
|
# An existing channel/fragment on the source URL is stripped and re-stamped.
|
||||||
|
assert build_listing_url("/properties/200#/?channel=RES_BUY", True) == (
|
||||||
|
"https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW"
|
||||||
|
)
|
||||||
|
# Missing URL stays empty.
|
||||||
|
assert build_listing_url("", True) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_rightmove_transform_tags_new_builds_res_new() -> None:
|
||||||
|
# The Rightmove search response marks new-build developments with
|
||||||
|
# development=True; transform_property must stamp the listing URL RES_NEW.
|
||||||
|
new_build = {**_rightmove_prop(), "development": True}
|
||||||
|
result = transform_property(new_build, "SW9", StubPostcodeIndex("SW9 7AA"))
|
||||||
|
assert result is not None
|
||||||
|
assert result["Listing URL"].endswith("#/?channel=RES_NEW")
|
||||||
|
|
||||||
|
# Ordinary resale (development absent/false) stays RES_BUY.
|
||||||
|
resale = transform_property(_rightmove_prop(), "SW9", StubPostcodeIndex("SW9 7AA"))
|
||||||
|
assert resale is not None
|
||||||
|
assert resale["Listing URL"].endswith("#/?channel=RES_BUY")
|
||||||
|
|
|
||||||
|
|
@ -103,7 +103,7 @@ def test_parse_detail_geo_merges_location_uprn_with_address_full_address() -> No
|
||||||
def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None:
|
def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None:
|
||||||
# The only `address` twin on the page belongs to a different uprn (a
|
# The only `address` twin on the page belongs to a different uprn (a
|
||||||
# comparable listing). With a uprn to match on, an unrelated twin is never
|
# comparable listing). With a uprn to match on, an unrelated twin is never
|
||||||
# borrowed — full_address stays None rather than grabbing the wrong street.
|
# borrowed: full_address stays None rather than grabbing the wrong street.
|
||||||
html = (
|
html = (
|
||||||
'"location":{"outcode":"NR29",'
|
'"location":{"outcode":"NR29",'
|
||||||
'"coordinates":{"latitude":52.716014,"longitude":1.614495},'
|
'"coordinates":{"latitude":52.716014,"longitude":1.614495},'
|
||||||
|
|
@ -185,7 +185,7 @@ def test_parse_detail_geo_returns_none_for_garbage() -> None:
|
||||||
assert parse_detail_geo("<html><body>no data here</body></html>") is None
|
assert parse_detail_geo("<html><body>no data here</body></html>") is None
|
||||||
assert parse_detail_geo("") is None
|
assert parse_detail_geo("") is None
|
||||||
# Coordinates that are not inside a property location/address wrapper (e.g.
|
# Coordinates that are not inside a property location/address wrapper (e.g.
|
||||||
# only an unwrapped POI) yield nothing — safe degradation to the outcode.
|
# only an unwrapped POI) yield nothing, safe degradation to the outcode.
|
||||||
assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None
|
assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -149,7 +149,7 @@ def map_property_type(sub_type: str | None) -> str:
|
||||||
return "Terraced"
|
return "Terraced"
|
||||||
if "house" in lower or "cottage" in lower:
|
if "house" in lower or "cottage" in lower:
|
||||||
return "Detached"
|
return "Detached"
|
||||||
log.warning("Unknown propertySubType: %r — mapping to Other", sub_type)
|
log.warning("Unknown propertySubType: %r, mapping to Other", sub_type)
|
||||||
return "Other"
|
return "Other"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -267,7 +267,7 @@ def build_register_address(
|
||||||
property's own number or name (e.g. Zoopla detail pages expose
|
property's own number or name (e.g. Zoopla detail pages expose
|
||||||
``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address
|
``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address
|
||||||
carries the house identifier that the EPC/Price-Paid register addresses also
|
carries the house identifier that the EPC/Price-Paid register addresses also
|
||||||
use — turning a fuzzy street match into a near-exact one. Falls back to the
|
use, turning a fuzzy street match into a near-exact one. Falls back to the
|
||||||
plain cleaned address when no number/name is available.
|
plain cleaned address when no number/name is available.
|
||||||
"""
|
"""
|
||||||
cleaned = clean_listing_address(raw_address)
|
cleaned = clean_listing_address(raw_address)
|
||||||
|
|
@ -282,6 +282,23 @@ def build_register_address(
|
||||||
return f"{number_or_name}, {cleaned}" if cleaned else number_or_name
|
return f"{number_or_name}, {cleaned}" if cleaned else number_or_name
|
||||||
|
|
||||||
|
|
||||||
|
def build_listing_url(property_url: str | None, is_new_build: bool) -> str:
|
||||||
|
"""Build the canonical Rightmove listing URL with an explicit channel marker.
|
||||||
|
|
||||||
|
The search API always echoes ``?channel=RES_BUY`` in ``propertyUrl`` even for
|
||||||
|
new-build developments (the request channel stays BUY), so the channel is
|
||||||
|
re-stamped here from the per-listing ``development`` flag: ``RES_NEW`` for new
|
||||||
|
builds, ``RES_BUY`` for ordinary resale. The map UI reads this marker to split
|
||||||
|
new vs non-new listings. Any channel/fragment already on ``propertyUrl`` is
|
||||||
|
stripped first so the result is deterministic.
|
||||||
|
"""
|
||||||
|
if not property_url:
|
||||||
|
return ""
|
||||||
|
base = property_url.split("#", 1)[0].split("?", 1)[0]
|
||||||
|
channel = "RES_NEW" if is_new_build else "RES_BUY"
|
||||||
|
return f"{RIGHTMOVE_BASE}{base}#/?channel={channel}"
|
||||||
|
|
||||||
|
|
||||||
def transform_property(
|
def transform_property(
|
||||||
prop: dict,
|
prop: dict,
|
||||||
outcode: str,
|
outcode: str,
|
||||||
|
|
@ -337,7 +354,7 @@ def transform_property(
|
||||||
|
|
||||||
inferred_postcode = pc_index.nearest(lat, lng)
|
inferred_postcode = pc_index.nearest(lat, lng)
|
||||||
if not inferred_postcode:
|
if not inferred_postcode:
|
||||||
log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng)
|
log.debug("No England postcode for property at %.4f, %.4f; skipping", lat, lng)
|
||||||
return None
|
return None
|
||||||
raw_address = prop.get("displayAddress", "") or ""
|
raw_address = prop.get("displayAddress", "") or ""
|
||||||
extracted_postcode = extract_full_postcode(raw_address)
|
extracted_postcode = extract_full_postcode(raw_address)
|
||||||
|
|
@ -391,7 +408,7 @@ def transform_property(
|
||||||
"price_frequency": "",
|
"price_frequency": "",
|
||||||
"Price qualifier": price_qualifier,
|
"Price qualifier": price_qualifier,
|
||||||
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
|
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
|
||||||
"Listing URL": RIGHTMOVE_BASE + property_url if property_url else "",
|
"Listing URL": build_listing_url(property_url, bool(prop.get("development"))),
|
||||||
"Listing features": key_features,
|
"Listing features": key_features,
|
||||||
"first_visible_date": prop.get("firstVisibleDate", ""),
|
"first_visible_date": prop.get("firstVisibleDate", ""),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""Zoopla scraping via FlareSolverr (no browser/VNC needed).
|
"""Zoopla scraping via FlareSolverr (no browser/VNC needed).
|
||||||
|
|
||||||
FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which
|
FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which
|
||||||
still contains the React Server Components flight stream — so the existing pure
|
still contains the React Server Components flight stream, so the existing pure
|
||||||
parsers work unchanged:
|
parsers work unchanged:
|
||||||
- the search page yields the outcode's listing detail URLs, and
|
- the search page yields the outcode's listing detail URLs, and
|
||||||
- each detail page's flight stream carries the property's location object
|
- each detail page's flight stream carries the property's location object
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import type { TFunction } from 'i18next';
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
import { cellToLatLng, polygonToCells } from 'h3-js';
|
import { cellToLatLng, polygonToCells } from 'h3-js';
|
||||||
import PriceHistoryChart from '../map/PriceHistoryChart';
|
import PriceHistoryChart from '../map/PriceHistoryChart';
|
||||||
|
import { MapErrorBoundary } from '../map/MapErrorBoundary';
|
||||||
import StackedBarChart from '../map/StackedBarChart';
|
import StackedBarChart from '../map/StackedBarChart';
|
||||||
import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions';
|
import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions';
|
||||||
import { DualHistogram } from '../map/DualHistogram';
|
import { DualHistogram } from '../map/DualHistogram';
|
||||||
|
|
@ -214,10 +215,13 @@ const SHOWCASE_MAP_START_VIEW: ViewState = {
|
||||||
bearing: 0,
|
bearing: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// End the Match fly-in over Greater London so the "match" step lands on the same
|
||||||
|
// place the Inspect (SW5 9AA) and Scout (SW5/SE22/N4) steps then drill into. The
|
||||||
|
// single-search narrative used to break by zooming to Birmingham then showing London.
|
||||||
const SHOWCASE_MAP_END_VIEW: ViewState = {
|
const SHOWCASE_MAP_END_VIEW: ViewState = {
|
||||||
longitude: -1.89,
|
longitude: -0.12,
|
||||||
latitude: 52.49,
|
latitude: 51.51,
|
||||||
zoom: 8.72,
|
zoom: 9.05,
|
||||||
pitch: 0,
|
pitch: 0,
|
||||||
bearing: 0,
|
bearing: 0,
|
||||||
};
|
};
|
||||||
|
|
@ -260,7 +264,12 @@ function buildShowcaseMapData(): HexagonData[] {
|
||||||
}
|
}
|
||||||
|
|
||||||
const SHOWCASE_MAP_DATA = buildShowcaseMapData();
|
const SHOWCASE_MAP_DATA = buildShowcaseMapData();
|
||||||
const SHOWCASE_MAP_TOTAL_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => sum + item.count, 0);
|
// Count only the hexes around Greater London so the Match card's figure matches the
|
||||||
|
// area it has zoomed to, instead of mislabelling the England-wide total as one city.
|
||||||
|
const SHOWCASE_LONDON_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => {
|
||||||
|
const nearLondon = Math.abs(item.lat - 51.5072) < 0.55 && Math.abs(item.lon + 0.1276) < 0.85;
|
||||||
|
return nearLondon ? sum + item.count : sum;
|
||||||
|
}, 0);
|
||||||
const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = [];
|
const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = [];
|
||||||
const EMPTY_SHOWCASE_POIS: POI[] = [];
|
const EMPTY_SHOWCASE_POIS: POI[] = [];
|
||||||
|
|
||||||
|
|
@ -601,37 +610,42 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) {
|
||||||
return (
|
return (
|
||||||
<div className="pointer-events-none relative h-full overflow-hidden bg-warm-100 dark:bg-navy-950/50">
|
<div className="pointer-events-none relative h-full overflow-hidden bg-warm-100 dark:bg-navy-950/50">
|
||||||
{shouldRenderMap && (
|
{shouldRenderMap && (
|
||||||
<Suspense fallback={<div className="h-full bg-navy-950/40" aria-hidden="true" />}>
|
// The full deck.gl/maplibre map sits above the fold; contain any WebGL
|
||||||
<ProductMap
|
// context-loss crash to this pane (and auto-recover) instead of letting it
|
||||||
data={SHOWCASE_MAP_DATA}
|
// bubble to the top-level boundary and blank the hero.
|
||||||
postcodeData={EMPTY_SHOWCASE_POSTCODES}
|
<MapErrorBoundary>
|
||||||
usePostcodeView={false}
|
<Suspense fallback={<div className="h-full bg-navy-950/40" aria-hidden="true" />}>
|
||||||
pois={EMPTY_SHOWCASE_POIS}
|
<ProductMap
|
||||||
onViewChange={noopViewChange}
|
data={SHOWCASE_MAP_DATA}
|
||||||
viewFeature={null}
|
postcodeData={EMPTY_SHOWCASE_POSTCODES}
|
||||||
colorRange={null}
|
usePostcodeView={false}
|
||||||
filterRange={null}
|
pois={EMPTY_SHOWCASE_POIS}
|
||||||
viewSource={null}
|
onViewChange={noopViewChange}
|
||||||
onCancelPin={() => {}}
|
viewFeature={null}
|
||||||
features={DEMO_FEATURES}
|
colorRange={null}
|
||||||
selectedHexagonId={null}
|
filterRange={null}
|
||||||
hoveredHexagonId={null}
|
viewSource={null}
|
||||||
onHexagonClick={noopHexagonClick}
|
onCancelPin={() => {}}
|
||||||
onHexagonHover={noopHexagonHover}
|
features={DEMO_FEATURES}
|
||||||
initialViewState={viewState}
|
selectedHexagonId={null}
|
||||||
theme="dark"
|
hoveredHexagonId={null}
|
||||||
screenshotMode
|
onHexagonClick={noopHexagonClick}
|
||||||
hideLegend
|
onHexagonHover={noopHexagonHover}
|
||||||
densityLabel={t('home.showcaseMatchingHomesLabel')}
|
initialViewState={viewState}
|
||||||
totalCount={SHOWCASE_MAP_TOTAL_COUNT}
|
theme="dark"
|
||||||
/>
|
screenshotMode
|
||||||
</Suspense>
|
hideLegend
|
||||||
|
densityLabel={t('home.showcaseMatchingHomesLabel')}
|
||||||
|
totalCount={SHOWCASE_LONDON_COUNT}
|
||||||
|
/>
|
||||||
|
</Suspense>
|
||||||
|
</MapErrorBoundary>
|
||||||
)}
|
)}
|
||||||
<div className="pointer-events-none absolute bottom-4 left-4 max-w-[16rem] rounded-md border border-white/15 bg-navy-950/95 px-4 py-3 text-white shadow-2xl shadow-navy-950/35">
|
<div className="pointer-events-none absolute bottom-4 left-4 max-w-[16rem] rounded-md border border-white/15 bg-navy-950/95 px-4 py-3 text-white shadow-2xl shadow-navy-950/35">
|
||||||
<div className="text-sm font-black leading-none">Birmingham</div>
|
<div className="text-sm font-black leading-none">{t('home.showcaseStep2Region')}</div>
|
||||||
<div className="mt-1 text-xs font-bold leading-tight text-warm-200">
|
<div className="mt-1 text-xs font-bold leading-tight text-warm-200">
|
||||||
{t('home.showcaseMatchingHomes', {
|
{t('home.showcaseMatchingHomes', {
|
||||||
value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(),
|
value: SHOWCASE_LONDON_COUNT.toLocaleString(),
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
<div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400">
|
<div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400">
|
||||||
|
|
@ -789,19 +803,22 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
|
||||||
postcode: 'SW5 9AA',
|
postcode: 'SW5 9AA',
|
||||||
score: '94%',
|
score: '94%',
|
||||||
commute: t('home.showcaseMinutes', { count: 23 }),
|
commute: t('home.showcaseMinutes', { count: 23 }),
|
||||||
price: '£492k',
|
perSqm: '£8,200',
|
||||||
|
delta: '−16%',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
postcode: 'SE22 8EF',
|
postcode: 'SE22 8EF',
|
||||||
score: '91%',
|
score: '91%',
|
||||||
commute: t('home.showcaseMinutes', { count: 28 }),
|
commute: t('home.showcaseMinutes', { count: 28 }),
|
||||||
price: '£518k',
|
perSqm: '£5,900',
|
||||||
|
delta: '−24%',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
postcode: 'N4 2AB',
|
postcode: 'N4 2AB',
|
||||||
score: '88%',
|
score: '88%',
|
||||||
commute: t('home.showcaseMinutes', { count: 31 }),
|
commute: t('home.showcaseMinutes', { count: 31 }),
|
||||||
price: '£476k',
|
perSqm: '£6,400',
|
||||||
|
delta: '−21%',
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
@ -934,7 +951,12 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
|
||||||
<div className="truncate border-r border-warm-200 px-2 py-1.5 dark:border-navy-700 sm:px-3 sm:py-2.5">
|
<div className="truncate border-r border-warm-200 px-2 py-1.5 dark:border-navy-700 sm:px-3 sm:py-2.5">
|
||||||
{row.commute}
|
{row.commute}
|
||||||
</div>
|
</div>
|
||||||
<div className="truncate px-2 py-1.5 font-black sm:px-3 sm:py-2.5">{row.price}</div>
|
<div className="px-2 py-1.5 sm:px-3 sm:py-2.5">
|
||||||
|
<div className="truncate font-black tabular-nums">{row.perSqm}</div>
|
||||||
|
<div className="text-[10px] font-bold tabular-nums text-emerald-600 dark:text-emerald-300">
|
||||||
|
{row.delta}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -1031,6 +1053,7 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
|
||||||
const [isStagePaused, setIsStagePaused] = useState(false);
|
const [isStagePaused, setIsStagePaused] = useState(false);
|
||||||
const [hasStarted, setHasStarted] = useState(false);
|
const [hasStarted, setHasStarted] = useState(false);
|
||||||
const [canPauseOnHover, setCanPauseOnHover] = useState(false);
|
const [canPauseOnHover, setCanPauseOnHover] = useState(false);
|
||||||
|
const [prefersReducedMotion, setPrefersReducedMotion] = useState(false);
|
||||||
const showcaseRef = useRef<HTMLDivElement | null>(null);
|
const showcaseRef = useRef<HTMLDivElement | null>(null);
|
||||||
const inspectUserScrolledRef = useRef(false);
|
const inspectUserScrolledRef = useRef(false);
|
||||||
|
|
||||||
|
|
@ -1101,6 +1124,27 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps
|
||||||
return () => mediaQuery.removeEventListener('change', updateCanPause);
|
return () => mediaQuery.removeEventListener('change', updateCanPause);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (typeof window.matchMedia !== 'function') return;
|
||||||
|
const mediaQuery = window.matchMedia('(prefers-reduced-motion: reduce)');
|
||||||
|
const update = () => setPrefersReducedMotion(mediaQuery.matches);
|
||||||
|
update();
|
||||||
|
mediaQuery.addEventListener('change', update);
|
||||||
|
return () => mediaQuery.removeEventListener('change', update);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Under prefers-reduced-motion the progress-bar animation is disabled in CSS, so
|
||||||
|
// its onAnimationEnd (which normally advances the carousel) never fires and the
|
||||||
|
// demo would freeze on step 1. Drive the advance from a timer in that case.
|
||||||
|
useEffect(() => {
|
||||||
|
if (!prefersReducedMotion || !isProgressRunning) return;
|
||||||
|
const timer = window.setTimeout(
|
||||||
|
() => setActiveStep((step) => (step + 1) % SHOWCASE_STEP_COUNT),
|
||||||
|
activeStepIntervalMs
|
||||||
|
);
|
||||||
|
return () => window.clearTimeout(timer);
|
||||||
|
}, [prefersReducedMotion, isProgressRunning, activeStep, activeStepIntervalMs]);
|
||||||
|
|
||||||
const pauseForHover = () => {
|
const pauseForHover = () => {
|
||||||
if (canPauseOnHover) setIsStagePaused(true);
|
if (canPauseOnHover) setIsStagePaused(true);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -235,6 +235,7 @@ export interface HistoricalPrice {
|
||||||
year: number;
|
year: number;
|
||||||
month: number;
|
month: number;
|
||||||
price: number;
|
price: number;
|
||||||
|
is_new: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Property {
|
export interface Property {
|
||||||
|
|
@ -386,7 +387,7 @@ export interface HexagonStatsResponse {
|
||||||
* sector, shown alongside the national average for each crime metric. */
|
* sector, shown alongside the national average for each crime metric. */
|
||||||
crime_area_averages?: CrimeAreaAverage[];
|
crime_area_averages?: CrimeAreaAverage[];
|
||||||
/** Total individual crime records (last 7 years) across the selection's
|
/** Total individual crime records (last 7 years) across the selection's
|
||||||
* postcodes — the count behind the "individual crimes" list. */
|
* postcodes, the count behind the "individual crimes" list. */
|
||||||
crime_total_records?: number;
|
crime_total_records?: number;
|
||||||
central_postcode?: string;
|
central_postcode?: string;
|
||||||
/** Total usual residents (ONS Census 2021) across the postcodes in this
|
/** Total usual residents (ONS Census 2021) across the postcodes in this
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
Downloads five-year age band counts (TS007A) from the NOMIS API, then computes
|
Downloads five-year age band counts (TS007A) from the NOMIS API, then computes
|
||||||
the median age per LSOA using linear interpolation within the median class.
|
the median age per LSOA using linear interpolation within the median class.
|
||||||
|
|
||||||
Source: NOMIS (ONS Census 2021 — TS007A dataset, NM_2020_1)
|
Source: NOMIS (ONS Census 2021, TS007A dataset, NM_2020_1)
|
||||||
License: Open Government Licence v3.0
|
License: Open Government Licence v3.0
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -20,7 +20,7 @@ from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv
|
||||||
BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE"
|
BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE"
|
||||||
|
|
||||||
# Five-year age bands in order, with lower bounds for interpolation.
|
# Five-year age bands in order, with lower bounds for interpolation.
|
||||||
# The last band (85+) is open-ended — we treat it as 85-89 for median purposes.
|
# The last band (85+) is open-ended. We treat it as 85-89 for median purposes.
|
||||||
AGE_BANDS = [
|
AGE_BANDS = [
|
||||||
(0, 5), # Aged 0 to 4 years
|
(0, 5), # Aged 0 to 4 years
|
||||||
(5, 5), # Aged 5 to 9 years
|
(5, 5), # Aged 5 to 9 years
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU"
|
||||||
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
|
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
|
||||||
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
|
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
|
||||||
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
|
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
|
||||||
# to its own "DLR station" category — restoring DLR to the train/tube station
|
# to its own "DLR station" category, restoring DLR to the train/tube station
|
||||||
# list and giving it the DLR roundel rather than a generic tram icon.
|
# list and giving it the DLR roundel rather than a generic tram icon.
|
||||||
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
|
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
|
||||||
|
|
||||||
|
|
@ -337,7 +337,7 @@ class StationAccumulator:
|
||||||
# A single node is never both (ZZLU vs ZZDL), but a co-located
|
# A single node is never both (ZZLU vs ZZDL), but a co-located
|
||||||
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
|
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
|
||||||
# and DLR halves into one group carrying both flags; Tube is checked
|
# and DLR halves into one group carrying both flags; Tube is checked
|
||||||
# first so these resolve to "Tube station" — their primary identity —
|
# first so these resolve to "Tube station" (their primary identity),
|
||||||
# leaving "DLR station" for the DLR-only stops the fix targets.
|
# leaving "DLR station" for the DLR-only stops the fix targets.
|
||||||
if self.category == TRAM_METRO_CATEGORY and self.is_lu:
|
if self.category == TRAM_METRO_CATEGORY and self.is_lu:
|
||||||
return TUBE_STATION_CATEGORY
|
return TUBE_STATION_CATEGORY
|
||||||
|
|
|
||||||
|
|
@ -85,24 +85,24 @@ RESOLUTION = NATIVE_RESOLUTION
|
||||||
|
|
||||||
# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
|
# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
|
||||||
# otherwise inside the raster means "modelled below the lowest reporting band",
|
# otherwise inside the raster means "modelled below the lowest reporting band",
|
||||||
# i.e. genuinely quiet — see noise_overlay_tiles.py:167.
|
# i.e. genuinely quiet (see noise_overlay_tiles.py:167).
|
||||||
NOISE_NODATA_SENTINEL = np.float32(-96.0)
|
NOISE_NODATA_SENTINEL = np.float32(-96.0)
|
||||||
|
|
||||||
# Lowest modelled Defra Lden reporting band (dB). Verified against the actual
|
# Lowest modelled Defra Lden reporting band (dB). Verified against the actual
|
||||||
# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
|
# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
|
||||||
# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor
|
# (0, 40). Below the band, cells are encoded as 0.0 (genuinely quiet). We floor
|
||||||
# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
|
# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
|
||||||
# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
|
# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
|
||||||
# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
|
# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
|
||||||
# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
|
# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
|
||||||
# NOISE_COLOR_STOPS[0]) — a rendering threshold, not the data's reporting floor.
|
# NOISE_COLOR_STOPS[0]), a rendering threshold, not the data's reporting floor.
|
||||||
NOISE_QUIET_FLOOR_DB = np.float32(40.0)
|
NOISE_QUIET_FLOOR_DB = np.float32(40.0)
|
||||||
|
|
||||||
# Sample noise at the postcode representative point itself (no neighbourhood
|
# Sample noise at the postcode representative point itself (no neighbourhood
|
||||||
# window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha
|
# window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha
|
||||||
# of every postcode; because Defra road contours hug every modelled road and
|
# of every postcode; because Defra road contours hug every modelled road and
|
||||||
# representative points sit on/near streets, that inflated postcode noise by
|
# representative points sit on/near streets, that inflated postcode noise by
|
||||||
# roughly +9 dB (log scale) — making ~94% of England read >=55 dB Lden and
|
# roughly +9 dB (log scale), making ~94% of England read >=55 dB Lden and
|
||||||
# collapsing the metric's discrimination at the quiet end. Radius 0 ->
|
# collapsing the metric's discrimination at the quiet end. Radius 0 ->
|
||||||
# filter_size 1 -> the maximum_filter is skipped and each postcode reads the
|
# filter_size 1 -> the maximum_filter is skipped and each postcode reads the
|
||||||
# 10m cell it actually sits in.
|
# 10m cell it actually sits in.
|
||||||
|
|
@ -110,8 +110,8 @@ POSTCODE_NOISE_RADIUS_M = 0
|
||||||
|
|
||||||
# Adjacent download tiles overlap by the sampling radius so every postcode's
|
# Adjacent download tiles overlap by the sampling radius so every postcode's
|
||||||
# sampling footprint is fully contained in at least one tile. With point
|
# sampling footprint is fully contained in at least one tile. With point
|
||||||
# sampling (radius 0) this is 0 — a representative point falls inside exactly
|
# sampling (radius 0) this is 0 (a representative point falls inside exactly
|
||||||
# one tile — but the relationship is kept so any future non-zero radius keeps
|
# one tile), but the relationship is kept so any future non-zero radius keeps
|
||||||
# its window seam-safe.
|
# its window seam-safe.
|
||||||
TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
|
TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
|
||||||
|
|
||||||
|
|
@ -273,7 +273,7 @@ def _download_tile(
|
||||||
NoGeoTiffError,
|
NoGeoTiffError,
|
||||||
httpx.HTTPStatusError,
|
httpx.HTTPStatusError,
|
||||||
# TransportError is the superset of TimeoutException, ConnectError,
|
# TransportError is the superset of TimeoutException, ConnectError,
|
||||||
# ReadError and ProtocolError — including RemoteProtocolError, raised
|
# ReadError and ProtocolError, including RemoteProtocolError, raised
|
||||||
# when the WCS server closes the connection mid-stream ("incomplete
|
# when the WCS server closes the connection mid-stream ("incomplete
|
||||||
# chunked read"). All are transient; retry/split rather than letting
|
# chunked read"). All are transient; retry/split rather than letting
|
||||||
# one flaky tile crash the whole raster download.
|
# one flaky tile crash the whole raster download.
|
||||||
|
|
@ -446,7 +446,7 @@ def sample_noise_at_postcodes(
|
||||||
# Defra rasters encode TRUE nodata as the -96.0 sentinel (and
|
# Defra rasters encode TRUE nodata as the -96.0 sentinel (and
|
||||||
# occasionally non-finite / dataset.nodata); genuinely quiet ground
|
# occasionally non-finite / dataset.nodata); genuinely quiet ground
|
||||||
# below the model's lowest reporting band is encoded as 0.0. Only
|
# below the model's lowest reporting band is encoded as 0.0. Only
|
||||||
# the former is "we don't know" — the latter is a real "we know it's
|
# the former is "we don't know". The latter is a real "we know it's
|
||||||
# quiet" reading and must not collapse to null. So treat ONLY true
|
# quiet" reading and must not collapse to null. So treat ONLY true
|
||||||
# nodata as -inf (it never wins a max and never counts as coverage),
|
# nodata as -inf (it never wins a max and never counts as coverage),
|
||||||
# and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
|
# and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
|
||||||
|
|
@ -555,7 +555,7 @@ def main() -> None:
|
||||||
|
|
||||||
if not tile_paths:
|
if not tile_paths:
|
||||||
print(
|
print(
|
||||||
f"[{label}] WARNING: No tiles downloaded — column will be all null"
|
f"[{label}] WARNING: No tiles downloaded; column will be all null"
|
||||||
)
|
)
|
||||||
series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32)
|
series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ site id and the site's polygon centroid. Sites without access points fall
|
||||||
back to polygon centroids.
|
back to polygon centroids.
|
||||||
|
|
||||||
Using access points rather than polygon centroids gives much more accurate
|
Using access points rather than polygon centroids gives much more accurate
|
||||||
distance calculations — a property next to Hyde Park won't show 400m just
|
distance calculations: a property next to Hyde Park won't show 400m just
|
||||||
because the centroid is in the middle of the park. The site id / centroid
|
because the centroid is in the middle of the park. The site id / centroid
|
||||||
columns let downstream consumers (poi_proximity) collapse the frame back to
|
columns let downstream consumers (poi_proximity) collapse the frame back to
|
||||||
one row per SITE for counting, so a park with 30 gates counts as one park.
|
one row per SITE for counting, so a park with 30 gates counts as one park.
|
||||||
|
|
@ -190,7 +190,7 @@ def download_greenspace(output: Path) -> None:
|
||||||
print(f"Reading {site_shps[0].name} for function types...")
|
print(f"Reading {site_shps[0].name} for function types...")
|
||||||
site_funcs = _read_site_functions(site_shps[0])
|
site_funcs = _read_site_functions(site_shps[0])
|
||||||
|
|
||||||
# Step 2: Read access points (primary — park entrances)
|
# Step 2: Read access points (primary: park entrances)
|
||||||
print(f"Reading {access_shps[0].name}...")
|
print(f"Reading {access_shps[0].name}...")
|
||||||
ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points(
|
ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points(
|
||||||
access_shps[0], site_funcs
|
access_shps[0], site_funcs
|
||||||
|
|
|
||||||
|
|
@ -930,7 +930,7 @@ def main() -> None:
|
||||||
df.write_parquet(args.output)
|
df.write_parquet(args.output)
|
||||||
print(f"Saved to {args.output}")
|
print(f"Saved to {args.output}")
|
||||||
else:
|
else:
|
||||||
print("No places found — skipping output")
|
print("No places found, skipping output")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ def _data_rows(df: pl.DataFrame) -> pl.DataFrame:
|
||||||
|
|
||||||
The preamble length varies (title, optional "This worksheet contains..."
|
The preamble length varies (title, optional "This worksheet contains..."
|
||||||
note, then the header row starting with "Time period"), so locate the
|
note, then the header row starting with "Time period"), so locate the
|
||||||
header by content instead of counting rows — a fixed slice leaves the
|
header by content instead of counting rows: a fixed slice leaves the
|
||||||
header in the data whenever ONS adds or removes a note line.
|
header in the data whenever ONS adds or removes a note line.
|
||||||
"""
|
"""
|
||||||
header_marker = (
|
header_marker = (
|
||||||
|
|
@ -78,7 +78,7 @@ def _latest_rents_long(df: pl.DataFrame) -> pl.DataFrame:
|
||||||
print(f"LAs in latest month: {df.height}")
|
print(f"LAs in latest month: {df.height}")
|
||||||
|
|
||||||
# Melt to long format: one row per area x bedroom count.
|
# Melt to long format: one row per area x bedroom count.
|
||||||
# PIPR has no Studio category — one-bed rent used as proxy for bedrooms=0.
|
# PIPR has no Studio category: one-bed rent used as proxy for bedrooms=0.
|
||||||
frames = []
|
frames = []
|
||||||
for col, bedrooms in [
|
for col, bedrooms in [
|
||||||
("rent_1bed", 0), # Studio (proxy)
|
("rent_1bed", 0), # Studio (proxy)
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@
|
||||||
|
|
||||||
Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9)
|
Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9)
|
||||||
from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf
|
from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf
|
||||||
categories into our 3 output buckets — Owner occupied / Social rent /
|
categories into our 3 output buckets (Owner occupied / Social rent /
|
||||||
Private rent — emitting one row per LSOA with the percentage of households in
|
Private rent), emitting one row per LSOA with the percentage of households in
|
||||||
each. The three buckets sum to 100%, so downstream they render as a
|
each. The three buckets sum to 100%, so downstream they render as a
|
||||||
composition/ratio (like the ethnicity and qualifications stacked bars) AND each
|
composition/ratio (like the ethnicity and qualifications stacked bars) AND each
|
||||||
percentage is independently filterable.
|
percentage is independently filterable.
|
||||||
|
|
@ -19,7 +19,7 @@ NOTE this table counts HOUSEHOLDS (not usual residents). The join key downstream
|
||||||
(merge.py) is `lsoa21`, the same key used for ethnicity, qualifications,
|
(merge.py) is `lsoa21`, the same key used for ethnicity, qualifications,
|
||||||
median age, and IoD.
|
median age, and IoD.
|
||||||
|
|
||||||
Source: NOMIS (ONS Census 2021 — TS054 dataset, NM_2072_1)
|
Source: NOMIS (ONS Census 2021, TS054 dataset, NM_2072_1)
|
||||||
License: Open Government Licence v3.0
|
License: Open Government Licence v3.0
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -72,7 +72,7 @@ def test_ethnicity_routes_chinese_to_east_and_other_asian_to_se():
|
||||||
|
|
||||||
|
|
||||||
def test_ethnicity_percentages_independent_per_lsoa():
|
def test_ethnicity_percentages_independent_per_lsoa():
|
||||||
"""Two LSOAs get independent profiles — the LSOA granularity is the point."""
|
"""Two LSOAs get independent profiles: the LSOA granularity is the point."""
|
||||||
df = pl.concat(
|
df = pl.concat(
|
||||||
[
|
[
|
||||||
pl.DataFrame(
|
pl.DataFrame(
|
||||||
|
|
|
||||||
|
|
@ -281,7 +281,7 @@ def test_sample_noise_preserves_genuine_reading_above_quiet_floor(
|
||||||
|
|
||||||
# The lowest Defra reporting band is 40.0 dB; genuine readings populate
|
# The lowest Defra reporting band is 40.0 dB; genuine readings populate
|
||||||
# [40, ~80]. A genuine in-coverage reading at or just above the floor must be
|
# [40, ~80]. A genuine in-coverage reading at or just above the floor must be
|
||||||
# PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A
|
# PRESERVED, not clamped UP to the floor. Only true-quiet 0.0 is floored. A
|
||||||
# quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
|
# quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
|
||||||
# dB readings; this pins that they survive unchanged.
|
# dB readings; this pins that they survive unchanged.
|
||||||
floor = float(noise.NOISE_QUIET_FLOOR_DB)
|
floor = float(noise.NOISE_QUIET_FLOOR_DB)
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,7 @@ ENGLAND_PBF_URL = (
|
||||||
"https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf"
|
"https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Bus Open Data Service — pre-converted GTFS covering all England bus/tram/ferry
|
# Bus Open Data Service: pre-converted GTFS covering all England bus/tram/ferry
|
||||||
BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
|
BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
|
||||||
|
|
||||||
# National Rail Open Data API
|
# National Rail Open Data API
|
||||||
|
|
@ -597,7 +597,7 @@ def validate_gtfs_feed(
|
||||||
fail("has neither calendar.txt nor calendar_dates.txt")
|
fail("has neither calendar.txt nor calendar_dates.txt")
|
||||||
if not _calendar_active_in_window(z, names, window_start, window_end):
|
if not _calendar_active_in_window(z, names, window_start, window_end):
|
||||||
fail(
|
fail(
|
||||||
f"no service active between {window_start} and {window_end} — "
|
f"no service active between {window_start} and {window_end}: "
|
||||||
"the feed's calendars are stale/expired and it would contribute "
|
"the feed's calendars are stale/expired and it would contribute "
|
||||||
"zero service to routing"
|
"zero service to routing"
|
||||||
)
|
)
|
||||||
|
|
@ -697,7 +697,7 @@ def _iter_stop_time_trips(lines, trip_id_idx: int):
|
||||||
dtd2mysql currently writes rows grouped by trip and ordered by
|
dtd2mysql currently writes rows grouped by trip and ordered by
|
||||||
stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a
|
stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a
|
||||||
trip_id reappearing later raises instead of silently scrambling trips);
|
trip_id reappearing later raises instead of silently scrambling trips);
|
||||||
within-trip order is NOT assumed — callers sort each group by its original
|
within-trip order is NOT assumed: callers sort each group by its original
|
||||||
stop_sequence.
|
stop_sequence.
|
||||||
"""
|
"""
|
||||||
current_trip: str | None = None
|
current_trip: str | None = None
|
||||||
|
|
@ -1175,7 +1175,7 @@ def main() -> None:
|
||||||
cif = download_national_rail_cif(raw_dir)
|
cif = download_national_rail_cif(raw_dir)
|
||||||
if cif is None:
|
if cif is None:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"National Rail timetable was not downloaded — set "
|
"National Rail timetable was not downloaded: set "
|
||||||
"NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at "
|
"NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at "
|
||||||
"https://opendata.nationalrail.co.uk/). National Rail heavy rail is "
|
"https://opendata.nationalrail.co.uk/). National Rail heavy rail is "
|
||||||
"required; without it the transit network models every train journey "
|
"required; without it the transit network models every train journey "
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
"""Robust GEOS overlay helpers.
|
"""Robust GEOS overlay helpers.
|
||||||
|
|
||||||
Overlay operations (union, difference, intersection) can raise a
|
Overlay operations (union, difference, intersection) can raise a
|
||||||
``GEOSException`` — most often ``TopologyException: side location conflict``,
|
``GEOSException``, most often ``TopologyException: side location conflict``,
|
||||||
``Ring edge missing``, or ``found non-noded intersection`` — on geometries that
|
``Ring edge missing``, or ``found non-noded intersection``, on geometries that
|
||||||
contain near-coincident or near-degenerate edges, or that are individually
|
contain near-coincident or near-degenerate edges, or that are individually
|
||||||
invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG
|
invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG
|
||||||
engine, handed a grid size, nodes every edge onto that grid and finishes where
|
engine, handed a grid size, nodes every edge onto that grid and finishes where
|
||||||
|
|
@ -14,8 +14,8 @@ learned the hard way from a crash:
|
||||||
1. **Never precision-reduce with the default mode.** ``set_precision``'s default
|
1. **Never precision-reduce with the default mode.** ``set_precision``'s default
|
||||||
``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that
|
``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that
|
||||||
re-raises the very ``side location conflict`` we are trying to escape. We push
|
re-raises the very ``side location conflict`` we are trying to escape. We push
|
||||||
the grid into the overlay via the ``grid_size`` argument instead — where
|
the grid into the overlay via the ``grid_size`` argument instead (where
|
||||||
OverlayNG nodes robustly — and only ever call ``set_precision`` in
|
OverlayNG nodes robustly) and only ever call ``set_precision`` in
|
||||||
``pointwise`` mode (pure coordinate rounding, which cannot raise).
|
``pointwise`` mode (pure coordinate rounding, which cannot raise).
|
||||||
2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties,
|
2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties,
|
||||||
pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape.
|
pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape.
|
||||||
|
|
@ -40,7 +40,7 @@ from shapely import GEOSException, make_valid, set_precision
|
||||||
from shapely.geometry import Polygon
|
from shapely.geometry import Polygon
|
||||||
from shapely.ops import unary_union
|
from shapely.ops import unary_union
|
||||||
|
|
||||||
# 0.1 mm in metres — well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
|
# 0.1 mm in metres: well below MIN_GEOM_AREA (0.01 m^2) and survey resolution.
|
||||||
_SNAP_GRID = 1e-4
|
_SNAP_GRID = 1e-4
|
||||||
|
|
||||||
_EMPTY = Polygon()
|
_EMPTY = Polygon()
|
||||||
|
|
|
||||||
|
|
@ -120,7 +120,7 @@ def _is_pointlike(geom_bng) -> bool:
|
||||||
def _rescue_footprint(geom_bng) -> dict | None:
|
def _rescue_footprint(geom_bng) -> dict | None:
|
||||||
"""Fatten a degenerate BNG geometry into a representable footprint and snap.
|
"""Fatten a degenerate BNG geometry into a representable footprint and snap.
|
||||||
|
|
||||||
A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon — the
|
A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon, the
|
||||||
signature of a tower-block postcode whose UPRNs all share one coordinate)
|
signature of a tower-block postcode whose UPRNs all share one coordinate)
|
||||||
gets a building-scale buffer so it is not reduced to an invisible sub-metre
|
gets a building-scale buffer so it is not reduced to an invisible sub-metre
|
||||||
dot; thin slivers that still carry length keep the minimal buffer.
|
dot; thin slivers that still carry length keep the minimal buffer.
|
||||||
|
|
@ -263,7 +263,7 @@ def merge_fragments(
|
||||||
# Close tiny gaps between adjacent OA boundary edges (float mismatches).
|
# Close tiny gaps between adjacent OA boundary edges (float mismatches).
|
||||||
# The closing can erode a tiny MultiPolygon (e.g. a postcode with only a
|
# The closing can erode a tiny MultiPolygon (e.g. a postcode with only a
|
||||||
# sliver fragment) to nothing, which would leave the postcode with no
|
# sliver fragment) to nothing, which would leave the postcode with no
|
||||||
# geometry at all — keep the un-closed shape if that happens.
|
# geometry at all. Keep the un-closed shape if that happens.
|
||||||
if combined.geom_type == "MultiPolygon":
|
if combined.geom_type == "MultiPolygon":
|
||||||
closed = combined.buffer(5.0).buffer(-5.0)
|
closed = combined.buffer(5.0).buffer(-5.0)
|
||||||
if not closed.is_valid:
|
if not closed.is_valid:
|
||||||
|
|
@ -308,7 +308,7 @@ def _polygonal(geom):
|
||||||
return None
|
return None
|
||||||
# Both callers run on WGS84-degree output geometry, so the robustness
|
# Both callers run on WGS84-degree output geometry, so the robustness
|
||||||
# fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre
|
# fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre
|
||||||
# default — a coarse metre grid would obliterate a degree-scale shape.
|
# default. A coarse metre grid would obliterate a degree-scale shape.
|
||||||
merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG)
|
merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG)
|
||||||
return merged if not merged.is_empty else None
|
return merged if not merged.is_empty else None
|
||||||
return None
|
return None
|
||||||
|
|
@ -324,7 +324,7 @@ def _resolve_overlaps(
|
||||||
containment (a postcode fully enclosed by another). Each postcode is trimmed
|
containment (a postcode fully enclosed by another). Each postcode is trimmed
|
||||||
by the union of its higher-priority overlapping neighbours, where **priority =
|
by the union of its higher-priority overlapping neighbours, where **priority =
|
||||||
ascending area**: a smaller postcode wins contested ground. That single rule
|
ascending area**: a smaller postcode wins contested ground. That single rule
|
||||||
handles both cases correctly — an enclosed postcode is always smaller than its
|
handles both cases correctly: an enclosed postcode is always smaller than its
|
||||||
container, so it keeps its area while the container gets a hole (a `overlaps`
|
container, so it keeps its area while the container gets a hole (a `overlaps`
|
||||||
query alone would miss containment entirely). Run last, on the final output
|
query alone would miss containment entirely). Run last, on the final output
|
||||||
geometries, so nothing re-introduces overlap afterwards. A postcode that would
|
geometries, so nothing re-introduces overlap afterwards. A postcode that would
|
||||||
|
|
@ -348,7 +348,7 @@ def _resolve_overlaps(
|
||||||
arr = np.array(geoms, dtype=object)
|
arr = np.array(geoms, dtype=object)
|
||||||
pairs: set[tuple[int, int]] = set()
|
pairs: set[tuple[int, int]] = set()
|
||||||
# "overlaps" gives partial overlaps; "contains" gives containment (which
|
# "overlaps" gives partial overlaps; "contains" gives containment (which
|
||||||
# "overlaps" excludes) — together they cover every 2-D overlap without the
|
# "overlaps" excludes). Together they cover every 2-D overlap without the
|
||||||
# edge-touch explosion a plain "intersects" query would add.
|
# edge-touch explosion a plain "intersects" query would add.
|
||||||
for predicate in ("overlaps", "contains"):
|
for predicate in ("overlaps", "contains"):
|
||||||
qsrc, qtgt = tree.query(arr, predicate=predicate)
|
qsrc, qtgt = tree.query(arr, predicate=predicate)
|
||||||
|
|
@ -577,7 +577,7 @@ def _grid_footprint(geom):
|
||||||
pass can shave a small (e.g. co-located, non-geographic) postcode down to a
|
pass can shave a small (e.g. co-located, non-geographic) postcode down to a
|
||||||
sub-grid sliver that disappears when snapped to output precision. Rather than
|
sub-grid sliver that disappears when snapped to output precision. Rather than
|
||||||
drop it, place a minimal valid footprint at its location. The tiny overlap
|
drop it, place a minimal valid footprint at its location. The tiny overlap
|
||||||
this re-creates with the neighbour that trimmed it is harmless — the output
|
this re-creates with the neighbour that trimmed it is harmless: the output
|
||||||
partition is best-effort, a missing boundary is a hard validation failure.
|
partition is best-effort, a missing boundary is a hard validation failure.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from .voronoi import compute_voronoi_regions
|
||||||
MIN_GEOM_AREA = 0.01
|
MIN_GEOM_AREA = 0.01
|
||||||
|
|
||||||
# Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a
|
# Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a
|
||||||
# crowded multi-postcode OA — its Voronoi cell ∩ remaining collapses below
|
# crowded multi-postcode OA: its Voronoi cell ∩ remaining collapses below
|
||||||
# MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a
|
# MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a
|
||||||
# co-located postcode. Every *active* postcode must keep a boundary
|
# co-located postcode. Every *active* postcode must keep a boundary
|
||||||
# (validate_outputs is zero-tolerance), so it gets a small disc at its true seed
|
# (validate_outputs is zero-tolerance), so it gets a small disc at its true seed
|
||||||
|
|
@ -77,7 +77,7 @@ def process_oa(
|
||||||
fragments.append((pc, merged))
|
fragments.append((pc, merged))
|
||||||
|
|
||||||
# Every postcode with a UPRN seed in this OA must keep at least a minimal
|
# Every postcode with a UPRN seed in this OA must keep at least a minimal
|
||||||
# footprint — in a dense OA (a block of flats with hundreds of distinct
|
# footprint: in a dense OA (a block of flats with hundreds of distinct
|
||||||
# postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or
|
# postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or
|
||||||
# be fully absorbed by a co-located postcode's INSPIRE parcel, producing no
|
# be fully absorbed by a co-located postcode's INSPIRE parcel, producing no
|
||||||
# fragment, and an active postcode must never be dropped.
|
# fragment, and an active postcode must never be dropped.
|
||||||
|
|
@ -142,7 +142,7 @@ def _claim_inspire_parcels(
|
||||||
# UPRNs from a single postcode goes wholly to that postcode. A parcel shared
|
# UPRNs from a single postcode goes wholly to that postcode. A parcel shared
|
||||||
# by several postcodes (a block of flats spanning postcodes, or overlapping
|
# by several postcodes (a block of flats spanning postcodes, or overlapping
|
||||||
# parcel data) is split between them via a sub-Voronoi over their own UPRNs
|
# parcel data) is split between them via a sub-Voronoi over their own UPRNs
|
||||||
# clipped to the parcel — so EVERY contained postcode keeps part of the
|
# clipped to the parcel, so EVERY contained postcode keeps part of the
|
||||||
# parcel. A bare majority vote would hand the whole parcel to one winner and
|
# parcel. A bare majority vote would hand the whole parcel to one winner and
|
||||||
# leave the losers' UPRNs trapped inside claimed land, dropping them from
|
# leave the losers' UPRNs trapped inside claimed land, dropping them from
|
||||||
# both this claim and the `remaining` polygon handed to Voronoi downstream.
|
# both this claim and the `remaining` polygon handed to Voronoi downstream.
|
||||||
|
|
@ -312,7 +312,7 @@ def _extract_polygonal(geom) -> Polygon | MultiPolygon | None:
|
||||||
return polys[0]
|
return polys[0]
|
||||||
# Union (not bare MultiPolygon construction): make_valid can emit
|
# Union (not bare MultiPolygon construction): make_valid can emit
|
||||||
# overlapping polygonal parts, and a MultiPolygon of overlapping parts is
|
# overlapping polygonal parts, and a MultiPolygon of overlapping parts is
|
||||||
# invalid — it double-counts area and makes the next `.difference()` raise
|
# invalid: it double-counts area and makes the next `.difference()` raise
|
||||||
# a TopologyException that aborts the OA (and, in parallel mode, the
|
# a TopologyException that aborts the OA (and, in parallel mode, the
|
||||||
# worker). safe_union merges them into a valid geometry.
|
# worker). safe_union merges them into a valid geometry.
|
||||||
merged = safe_union(polys)
|
merged = safe_union(polys)
|
||||||
|
|
|
||||||
|
|
@ -82,7 +82,7 @@ def load_uprns(
|
||||||
# Remap terminated postcodes to their nearest active successor. The
|
# Remap terminated postcodes to their nearest active successor. The
|
||||||
# successor generally lives in a DIFFERENT OA (and at different grid
|
# successor generally lives in a DIFFERENT OA (and at different grid
|
||||||
# coordinates), so the remapped point must adopt the successor's
|
# coordinates), so the remapped point must adopt the successor's
|
||||||
# authoritative OA/coords — keeping the terminated postcode's original
|
# authoritative OA/coords. Keeping the terminated postcode's original
|
||||||
# OA would seed the successor into an OA it doesn't belong to, splitting
|
# OA would seed the successor into an OA it doesn't belong to, splitting
|
||||||
# its boundary across OAs. Genuine (non-remapped) UPRN rows keep their
|
# its boundary across OAs. Genuine (non-remapped) UPRN rows keep their
|
||||||
# own OA, since a live postcode can legitimately span several OAs.
|
# own OA, since a live postcode can legitimately span several OAs.
|
||||||
|
|
@ -127,7 +127,7 @@ def load_uprns(
|
||||||
uprns.sort("OA21CD").sink_parquet(tmp_path)
|
uprns.sort("OA21CD").sink_parquet(tmp_path)
|
||||||
release_memory()
|
release_memory()
|
||||||
|
|
||||||
# Read the sorted data — only one copy in memory (~2GB)
|
# Read the sorted data: only one copy in memory (~2GB)
|
||||||
df = pl.read_parquet(tmp_path)
|
df = pl.read_parquet(tmp_path)
|
||||||
tmp_path.unlink()
|
tmp_path.unlink()
|
||||||
n = len(df)
|
n = len(df)
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ Stratified by property type and postcode sector, with IRLS Huber regression,
|
||||||
hierarchical shrinkage (sector → district → area → national → hedonic),
|
hierarchical shrinkage (sector → district → area → national → hedonic),
|
||||||
and KD-tree spatial smoothing for sparse sectors.
|
and KD-tree spatial smoothing for sparse sectors.
|
||||||
|
|
||||||
Output: price_index.parquet — sector x type_group x year -> log_index
|
Output: price_index.parquet (sector x type_group x year -> log_index)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
|
||||||
# Natural join key from estimates back onto properties: postcode plus the
|
# Natural join key from estimates back onto properties: postcode plus the
|
||||||
# coalesced register/EPC address. This is unique and non-null on the deduped
|
# coalesced register/EPC address. This is unique and non-null on the deduped
|
||||||
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
|
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
|
||||||
# estimates 1:1 onto properties regardless of row order — estimates are computed
|
# estimates 1:1 onto properties regardless of row order. Estimates are computed
|
||||||
# from a separate price_inputs.parquet, so a positional key would not line up.
|
# from a separate price_inputs.parquet, so a positional key would not line up.
|
||||||
JOIN_ADDRESS = "_join_address"
|
JOIN_ADDRESS = "_join_address"
|
||||||
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
|
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
|
||||||
|
|
|
||||||
|
|
@ -315,7 +315,7 @@ def test_run_tenure_history_tracks_rent_owner_transitions(tmp_path: Path):
|
||||||
|
|
||||||
def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
|
def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
|
||||||
# A property only ever observed as owner-occupied has no tenure change worth
|
# A property only ever observed as owner-occupied has no tenure change worth
|
||||||
# surfacing — the timeline column is null (no events), not a noisy baseline.
|
# surfacing: the timeline column is null (no events), not a noisy baseline.
|
||||||
zip_path = tmp_path / "domestic-csv.zip"
|
zip_path = tmp_path / "domestic-csv.zip"
|
||||||
_write_epc_zip(
|
_write_epc_zip(
|
||||||
zip_path,
|
zip_path,
|
||||||
|
|
@ -339,6 +339,37 @@ def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
|
||||||
assert df.get_column("tenure_history").to_list() == [None]
|
assert df.get_column("tenure_history").to_list() == [None]
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_latest_tenure_status_reflects_most_recent_certificate(tmp_path: Path):
|
||||||
|
# Two certificates for one dwelling: an older social-rented cert and a newer
|
||||||
|
# owner-occupied one. The published latest_tenure_status must carry the
|
||||||
|
# LATEST certificate's normalized tenure ("Owner-occupied"), while
|
||||||
|
# was_council_house stays "Yes" because the dwelling was social at some
|
||||||
|
# point. This also confirms latest_tenure_status reaches the epc_pp parquet.
|
||||||
|
zip_path = tmp_path / "domestic-csv.zip"
|
||||||
|
_write_epc_zip(
|
||||||
|
zip_path,
|
||||||
|
[
|
||||||
|
_row(inspection_date="2016-04-01", tenure="Rented (social)"),
|
||||||
|
_row(inspection_date="2024-04-01", tenure="owner-occupied"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
price_paid_path = tmp_path / "price-paid.parquet"
|
||||||
|
_price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet(
|
||||||
|
price_paid_path
|
||||||
|
)
|
||||||
|
|
||||||
|
output_path = tmp_path / "epc-pp.parquet"
|
||||||
|
_run(zip_path, price_paid_path, output_path, tmp_path)
|
||||||
|
|
||||||
|
df = pl.read_parquet(output_path)
|
||||||
|
|
||||||
|
assert df.height == 1
|
||||||
|
assert df.select("latest_tenure_status", "was_council_house").to_dicts() == [
|
||||||
|
{"latest_tenure_status": "Owner-occupied", "was_council_house": "Yes"}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path):
|
def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path):
|
||||||
# Two certificates for the same property. The cert with the garbled,
|
# Two certificates for the same property. The cert with the garbled,
|
||||||
# unparseable inspection_date must NOT be chosen as "latest": a string sort
|
# unparseable inspection_date must NOT be chosen as "latest": a string sort
|
||||||
|
|
@ -565,7 +596,7 @@ def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: P
|
||||||
price_paid_path = tmp_path / "price-paid.parquet"
|
price_paid_path = tmp_path / "price-paid.parquet"
|
||||||
pl.DataFrame(
|
pl.DataFrame(
|
||||||
{
|
{
|
||||||
# 5_000 is below MIN_PRICE (10_000) — a nominal/junk transfer that
|
# 5_000 is below MIN_PRICE (10_000), a nominal/junk transfer that
|
||||||
# must still anchor the construction year but stay out of the price
|
# must still anchor the construction year but stay out of the price
|
||||||
# aggregations.
|
# aggregations.
|
||||||
"price": [5_000, 300_000],
|
"price": [5_000, 300_000],
|
||||||
|
|
@ -603,7 +634,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
|
||||||
# lands AFTER its first Land Registry sale (1998). A dwelling cannot have
|
# lands AFTER its first Land Registry sale (1998). A dwelling cannot have
|
||||||
# been built after it was first sold, so the published build year must be
|
# been built after it was first sold, so the published build year must be
|
||||||
# capped at the first transfer year (1998), not the later band estimate.
|
# capped at the first transfer year (1998), not the later band estimate.
|
||||||
# It stays flagged as an estimate (approximate=1) — it is still EPC-derived.
|
# It stays flagged as an estimate (approximate=1). It is still EPC-derived.
|
||||||
zip_path = tmp_path / "domestic-csv.zip"
|
zip_path = tmp_path / "domestic-csv.zip"
|
||||||
_write_epc_zip(
|
_write_epc_zip(
|
||||||
zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")]
|
zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")]
|
||||||
|
|
@ -627,7 +658,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
|
||||||
|
|
||||||
def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
|
def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
|
||||||
# The common case: the EPC band (1950-1966 -> 1958) predates the first
|
# The common case: the EPC band (1950-1966 -> 1958) predates the first
|
||||||
# recorded sale (2020). The cap must NOT fire — the band estimate stands.
|
# recorded sale (2020). The cap must NOT fire. The band estimate stands.
|
||||||
zip_path = tmp_path / "domestic-csv.zip"
|
zip_path = tmp_path / "domestic-csv.zip"
|
||||||
_write_epc_zip(zip_path)
|
_write_epc_zip(zip_path)
|
||||||
|
|
||||||
|
|
@ -649,7 +680,7 @@ def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
|
||||||
def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
|
def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
|
||||||
# A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the
|
# A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the
|
||||||
# NEW floor (10k): it must now be RETAINED in the price aggregations. This
|
# NEW floor (10k): it must now be RETAINED in the price aggregations. This
|
||||||
# pins the 50k->10k change — it fails on the pre-fix 50k floor (where 30k was
|
# pins the 50k->10k change: it fails on the pre-fix 50k floor (where 30k was
|
||||||
# excluded, giving historical_prices length 1 / latest_price 250_000).
|
# excluded, giving historical_prices length 1 / latest_price 250_000).
|
||||||
zip_path = tmp_path / "domestic-csv.zip"
|
zip_path = tmp_path / "domestic-csv.zip"
|
||||||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||||
|
|
@ -748,8 +779,8 @@ def test_run_collapses_duplicate_transactions(tmp_path: Path):
|
||||||
# The duplicated 250_000 sale collapses to one entry; two distinct sales.
|
# The duplicated 250_000 sale collapses to one entry; two distinct sales.
|
||||||
assert df.get_column("historical_prices").to_list() == [
|
assert df.get_column("historical_prices").to_list() == [
|
||||||
[
|
[
|
||||||
{"year": 2020, "month": 2, "price": 200_000},
|
{"year": 2020, "month": 2, "price": 200_000, "is_new": False},
|
||||||
{"year": 2024, "month": 2, "price": 250_000},
|
{"year": 2024, "month": 2, "price": 250_000, "is_new": False},
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
assert df.get_column("latest_price").to_list() == [250_000]
|
assert df.get_column("latest_price").to_list() == [250_000]
|
||||||
|
|
@ -777,7 +808,7 @@ def test_run_excludes_implausible_price_jump_but_keeps_property(tmp_path: Path):
|
||||||
assert df.height == 1
|
assert df.height == 1
|
||||||
assert df.get_column("latest_price").to_list() == [140_000]
|
assert df.get_column("latest_price").to_list() == [140_000]
|
||||||
assert df.get_column("historical_prices").to_list() == [
|
assert df.get_column("historical_prices").to_list() == [
|
||||||
[{"year": 2016, "month": 6, "price": 140_000}]
|
[{"year": 2016, "month": 6, "price": 140_000, "is_new": False}]
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -593,7 +593,7 @@ def test_transform_grocery_dedup_drops_only_grocery_aspect(tmp_path):
|
||||||
# The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at
|
# The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at
|
||||||
# (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name
|
# (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name
|
||||||
# is the same physical store, so its Convenience Store (Groceries) row is a
|
# is the same physical store, so its Convenience Store (Groceries) row is a
|
||||||
# duplicate and must be dropped — but its NON-grocery aspect (a Post Office
|
# duplicate and must be dropped, but its NON-grocery aspect (a Post Office
|
||||||
# sharing the same OSM id) must survive. An independent shop away from the
|
# sharing the same OSM id) must survive. An independent shop away from the
|
||||||
# GEOLYTIX point keeps its grocery row.
|
# GEOLYTIX point keeps its grocery row.
|
||||||
raw = pl.DataFrame(
|
raw = pl.DataFrame(
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ DROP_CATEGORIES = {
|
||||||
"emergency/water_tank",
|
"emergency/water_tank",
|
||||||
"leisure/bleachers",
|
"leisure/bleachers",
|
||||||
"leisure/schoolyard",
|
"leisure/schoolyard",
|
||||||
# Park "furniture" / incidental features — not parks; they massively
|
# Park "furniture" / incidental features, not parks; they massively
|
||||||
# inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k).
|
# inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k).
|
||||||
"leisure/bandstand",
|
"leisure/bandstand",
|
||||||
"leisure/bird_hide",
|
"leisure/bird_hide",
|
||||||
|
|
@ -222,7 +222,7 @@ DROP_CATEGORIES = {
|
||||||
"public_transport/entrance",
|
"public_transport/entrance",
|
||||||
"public_transport/station",
|
"public_transport/station",
|
||||||
"public_transport/stop_position",
|
"public_transport/stop_position",
|
||||||
# Education amenities — schools come from GIAS instead. OSM coverage for
|
# Education amenities. Schools come from GIAS instead. OSM coverage for
|
||||||
# tertiary education, tutoring, and childcare is too noisy/incomplete to be
|
# tertiary education, tutoring, and childcare is too noisy/incomplete to be
|
||||||
# useful on a property-search map.
|
# useful on a property-search map.
|
||||||
"amenity/school",
|
"amenity/school",
|
||||||
|
|
@ -398,7 +398,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
||||||
"tourism/theme_park",
|
"tourism/theme_park",
|
||||||
# bicycle_rental/boat_rental/marina/slipway used to live here and
|
# bicycle_rental/boat_rental/marina/slipway used to live here and
|
||||||
# made up ~46% of the bucket (cycle-hire docks, boat ramps); they
|
# made up ~46% of the bucket (cycle-hire docks, boat ramps); they
|
||||||
# are infrastructure, not entertainment venues — see DROP_CATEGORIES.
|
# are infrastructure, not entertainment venues. See DROP_CATEGORIES.
|
||||||
"leisure/hackerspace",
|
"leisure/hackerspace",
|
||||||
"leisure/yes",
|
"leisure/yes",
|
||||||
],
|
],
|
||||||
|
|
@ -722,7 +722,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
||||||
[
|
[
|
||||||
"leisure/fitness_centre",
|
"leisure/fitness_centre",
|
||||||
# leisure/fitness_station (outdoor pull-up bars / trim-trail
|
# leisure/fitness_station (outdoor pull-up bars / trim-trail
|
||||||
# apparatus, ~2.5k) is not a gym — see DROP_CATEGORIES.
|
# apparatus, ~2.5k) is not a gym. See DROP_CATEGORIES.
|
||||||
"amenity/dojo",
|
"amenity/dojo",
|
||||||
"amenity/dancing_school",
|
"amenity/dancing_school",
|
||||||
],
|
],
|
||||||
|
|
@ -849,8 +849,8 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
||||||
"healthcare/pharmacy",
|
"healthcare/pharmacy",
|
||||||
"shop/chemist",
|
"shop/chemist",
|
||||||
# healthcare/alternative, shop/herbalist and shop/health (homeopaths,
|
# healthcare/alternative, shop/herbalist and shop/health (homeopaths,
|
||||||
# herbalists, generic "health" shops) are not dispensing pharmacies
|
# herbalists, generic "health" shops) are not dispensing pharmacies.
|
||||||
# — see DROP_CATEGORIES.
|
# See DROP_CATEGORIES.
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
# "Hospital & Clinic" used to be one bucket; an actual hospital and a small
|
# "Hospital & Clinic" used to be one bucket; an actual hospital and a small
|
||||||
|
|
@ -878,7 +878,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
||||||
"healthcare/laboratory",
|
"healthcare/laboratory",
|
||||||
"healthcare/rehabilitation",
|
"healthcare/rehabilitation",
|
||||||
"healthcare/vaccination_centre",
|
"healthcare/vaccination_centre",
|
||||||
# healthcare/yes (untyped junk rows) is dropped — see DROP_CATEGORIES.
|
# healthcare/yes (untyped junk rows) is dropped. See DROP_CATEGORIES.
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
|
|
@ -950,7 +950,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [
|
||||||
[
|
[
|
||||||
"tourism/gallery",
|
"tourism/gallery",
|
||||||
# tourism/artwork (statues, murals, village signs) was 93% of this
|
# tourism/artwork (statues, murals, village signs) was 93% of this
|
||||||
# bucket and is not a visitable gallery — see DROP_CATEGORIES.
|
# bucket and is not a visitable gallery. See DROP_CATEGORIES.
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
|
|
@ -1420,7 +1420,7 @@ def _school_icon_category_expr() -> pl.Expr:
|
||||||
# GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed
|
# GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed
|
||||||
# primary") so we normalise before matching.
|
# primary") so we normalise before matching.
|
||||||
phase = pl.col("phase").str.to_lowercase()
|
phase = pl.col("phase").str.to_lowercase()
|
||||||
# gias._format_age_range emits three shapes: "<low>–<high>" (em-dash),
|
# gias._format_age_range emits three shapes: "<low>–<high>" (en-dash),
|
||||||
# "up to <high>" (high-only) and "<low>+" (low-only). Extract the leading
|
# "up to <high>" (high-only) and "<low>+" (low-only). Extract the leading
|
||||||
# integer as low and the trailing integer as high, then suppress the wrong
|
# integer as low and the trailing integer as high, then suppress the wrong
|
||||||
# end for the one-sided shapes so they don't collapse to a single bound.
|
# end for the one-sided shapes so they don't collapse to a single bound.
|
||||||
|
|
@ -1477,7 +1477,7 @@ def _load_ofsted_ratings(ofsted_path: Path) -> pl.LazyFrame:
|
||||||
the conventional Ofsted labels; when there is no usable graded result
|
the conventional Ofsted labels; when there is no usable graded result
|
||||||
(null/"Not judged", e.g. schools last seen under the post-2024 ungraded
|
(null/"Not judged", e.g. schools last seen under the post-2024 ungraded
|
||||||
report-card framework) we fall back to "Ungraded inspection overall outcome"
|
report-card framework) we fall back to "Ungraded inspection overall outcome"
|
||||||
so genuinely good/outstanding schools aren't dropped — mirroring
|
so genuinely good/outstanding schools aren't dropped, mirroring
|
||||||
school_catchments.classify_good_plus_schools. Remaining nulls drop out."""
|
school_catchments.classify_good_plus_schools. Remaining nulls drop out."""
|
||||||
grade_col = pl.col("Latest OEIF overall effectiveness")
|
grade_col = pl.col("Latest OEIF overall effectiveness")
|
||||||
# See school_catchments: the ungraded outcome carries "School remains Good"/
|
# See school_catchments: the ungraded outcome carries "School remains Good"/
|
||||||
|
|
@ -1682,7 +1682,7 @@ def osm_groceries_colocated_with_geolytix(
|
||||||
|
|
||||||
An OSM Groceries row is a duplicate when a GEOLYTIX point lies within
|
An OSM Groceries row is a duplicate when a GEOLYTIX point lies within
|
||||||
``radius_m`` metres AND that point's brand tokens (its ``category``, e.g.
|
``radius_m`` metres AND that point's brand tokens (its ``category``, e.g.
|
||||||
"Tesco", "Co-op", "M&S") are all present in the OSM row's name — i.e. the
|
"Tesco", "Co-op", "M&S") are all present in the OSM row's name, i.e. the
|
||||||
same physical branded store. Short brands like "M&S" match via
|
same physical branded store. Short brands like "M&S" match via
|
||||||
_GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept.
|
_GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept.
|
||||||
|
|
||||||
|
|
@ -1703,7 +1703,7 @@ def osm_groceries_colocated_with_geolytix(
|
||||||
osm_ids = osm_groceries["id"].to_list()
|
osm_ids = osm_groceries["id"].to_list()
|
||||||
osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()]
|
osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()]
|
||||||
|
|
||||||
# Equirectangular projection to metres around the shared mean latitude — at
|
# Equirectangular projection to metres around the shared mean latitude. At
|
||||||
# England's scale this is accurate to well under the dedup radius.
|
# England's scale this is accurate to well under the dedup radius.
|
||||||
mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat])))
|
mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat])))
|
||||||
cos_lat = float(np.cos(np.radians(mean_lat)))
|
cos_lat = float(np.cos(np.radians(mean_lat)))
|
||||||
|
|
@ -1860,7 +1860,7 @@ def transform(
|
||||||
)
|
)
|
||||||
# Scope the drop to the Groceries group: a single OSM object can also
|
# Scope the drop to the Groceries group: a single OSM object can also
|
||||||
# carry a non-grocery aspect (e.g. a convenience store that is also a
|
# carry a non-grocery aspect (e.g. a convenience store that is also a
|
||||||
# Post Office), which must survive — only its duplicate grocery row goes.
|
# Post Office), which must survive. Only its duplicate grocery row goes.
|
||||||
lf = lf.filter(
|
lf = lf.filter(
|
||||||
~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids))
|
~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids))
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,7 @@ POSTCODE_DENSITY_PERCENTILE_COL = "Tree canopy density percentile within {radius
|
||||||
POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)"
|
POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)"
|
||||||
POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)"
|
POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)"
|
||||||
|
|
||||||
# National Forest Inventory (NFI) woodland — the geometric complement of TOW.
|
# National Forest Inventory (NFI) woodland: the geometric complement of TOW.
|
||||||
# NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700.
|
# NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700.
|
||||||
# Field names are from the NFI Woodland England 2022 release; re-check on bumps.
|
# Field names are from the NFI Woodland England 2022 release; re-check on bumps.
|
||||||
NFI_CATEGORY_COL = "CATEGORY"
|
NFI_CATEGORY_COL = "CATEGORY"
|
||||||
|
|
@ -263,7 +263,7 @@ def _postcode_buffers(
|
||||||
return circles, shapely.STRtree(circles)
|
return circles, shapely.STRtree(circles)
|
||||||
|
|
||||||
|
|
||||||
# 0.1 mm in the BNG working CRS (EPSG:27700) — far below survey resolution; the
|
# 0.1 mm in the BNG working CRS (EPSG:27700), far below survey resolution; the
|
||||||
# same grid the postcode_boundaries overlay uses.
|
# same grid the postcode_boundaries overlay uses.
|
||||||
_OVERLAY_GRID_M = 1e-4
|
_OVERLAY_GRID_M = 1e-4
|
||||||
|
|
||||||
|
|
@ -274,13 +274,13 @@ def _robust_intersection_area(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
||||||
External Forest Research TOW/NFI polygons are occasionally invalid
|
External Forest Research TOW/NFI polygons are occasionally invalid
|
||||||
(self-intersections), and a single bad polygon makes the batched
|
(self-intersections), and a single bad polygon makes the batched
|
||||||
``shapely.intersection`` raise ``TopologyException: side location conflict``,
|
``shapely.intersection`` raise ``TopologyException: side location conflict``,
|
||||||
aborting the whole run. The fast path is the raw batched overlay — unchanged,
|
aborting the whole run. The fast path is the raw batched overlay (unchanged,
|
||||||
full-speed, when the data is clean — and only a failure triggers repair.
|
full-speed, when the data is clean), and only a failure triggers repair.
|
||||||
|
|
||||||
The repair deliberately uses a *plain* overlay rather than the fixed-precision
|
The repair deliberately uses a *plain* overlay rather than the fixed-precision
|
||||||
(``grid_size``) one: ``make_valid`` can emit a mixed-dimension
|
(``grid_size``) one: ``make_valid`` can emit a mixed-dimension
|
||||||
``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG
|
``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG
|
||||||
rejects with ``Overlay input is mixed-dimension`` — whereas a plain overlay
|
rejects with ``Overlay input is mixed-dimension``, whereas a plain overlay
|
||||||
accepts it, and its non-polygonal debris has zero area and is dropped by the
|
accepts it, and its non-polygonal debris has zero area and is dropped by the
|
||||||
``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate
|
``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate
|
||||||
snap (which never raises) collapses the near-coincident edges behind any
|
snap (which never raises) collapses the near-coincident edges behind any
|
||||||
|
|
@ -496,7 +496,7 @@ def _finalize_metrics(
|
||||||
if over_count:
|
if over_count:
|
||||||
print(
|
print(
|
||||||
f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were "
|
f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were "
|
||||||
"capped — indicates overlapping TOW/NFI canopy within the buffer"
|
"capped, indicating overlapping TOW/NFI canopy within the buffer"
|
||||||
)
|
)
|
||||||
|
|
||||||
mean_height = np.divide(
|
mean_height = np.divide(
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,7 @@ def test_custom_radius(pois):
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
# 0.01 km = 10m — only the POI at the exact same location should match
|
# 0.01 km = 10m: only the POI at the exact same location should match
|
||||||
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01)
|
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01)
|
||||||
# The Restaurant at (51.5074, -0.1278) is at distance 0
|
# The Restaurant at (51.5074, -0.1278) is at distance 0
|
||||||
assert result["restaurants_0km"][0] >= 1
|
assert result["restaurants_0km"][0] >= 1
|
||||||
|
|
@ -135,9 +135,9 @@ def test_min_distance_finds_nearest(postcodes, pois):
|
||||||
assert len(result) == 2
|
assert len(result) == 2
|
||||||
|
|
||||||
ec1a = result.filter(pl.col("postcode") == "EC1A 1BB")
|
ec1a = result.filter(pl.col("postcode") == "EC1A 1BB")
|
||||||
# Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278) — very close
|
# Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278), very close
|
||||||
assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m
|
assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m
|
||||||
# Restaurant is co-located — distance ~0
|
# Restaurant is co-located: distance ~0
|
||||||
assert ec1a["restaurants_nearest_km"][0] < 0.01
|
assert ec1a["restaurants_nearest_km"][0] < 0.01
|
||||||
|
|
||||||
# Far-away postcode should still get the global nearest distance.
|
# Far-away postcode should still get the global nearest distance.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue