diff --git a/finder/test_rightmove_concurrency.py b/finder/test_rightmove_concurrency.py index cadc3b1..3ef3671 100644 --- a/finder/test_rightmove_concurrency.py +++ b/finder/test_rightmove_concurrency.py @@ -14,7 +14,7 @@ def _prop(pid, pin=None): # --------------------------------------------------------------------------- -# _needs_detail_fetch — accurate-pin skip +# _needs_detail_fetch: accurate-pin skip # --------------------------------------------------------------------------- @@ -32,7 +32,7 @@ def test_needs_detail_fetch_disabled_always_fetches(monkeypatch): # --------------------------------------------------------------------------- -# _prime_detail_postcodes — worklist selection + concurrent fetch +# _prime_detail_postcodes: worklist selection + concurrent fetch # --------------------------------------------------------------------------- @@ -97,7 +97,7 @@ def test_prime_is_a_noop_when_disabled_or_cap_zero(monkeypatch): # --------------------------------------------------------------------------- -# _paginate — end-to-end (network stubbed): accurate pins fall back to +# _paginate end-to-end (network stubbed): accurate pins fall back to # coordinates, approximate pins use the detail postcode. # --------------------------------------------------------------------------- diff --git a/finder/test_scraper_concurrency.py b/finder/test_scraper_concurrency.py index 6f9bbe3..24df360 100644 --- a/finder/test_scraper_concurrency.py +++ b/finder/test_scraper_concurrency.py @@ -9,7 +9,7 @@ import zoopla # --------------------------------------------------------------------------- -# _run_sources — Zoopla inline, others in threads, failures isolated +# _run_sources: Zoopla inline, others in threads, failures isolated # --------------------------------------------------------------------------- @@ -85,13 +85,13 @@ def test_seed_and_save_detail_caches_round_trip(tmp_path): def test_seed_detail_caches_tolerates_missing_files(tmp_path): rightmove._detail_postcode_cache.clear() - # No file written yet — seeding must not raise and must leave cache empty. + # No file written yet: seeding must not raise and must leave cache empty. scraper._seed_detail_caches(["rightmove"], tmp_path) assert rightmove._detail_postcode_cache == {} # --------------------------------------------------------------------------- -# run_scrape — full orchestration wiring (sources stubbed, no network) +# run_scrape: full orchestration wiring (sources stubbed, no network) # --------------------------------------------------------------------------- diff --git a/finder/test_transform.py b/finder/test_transform.py index 1da0a7e..1750fca 100644 --- a/finder/test_transform.py +++ b/finder/test_transform.py @@ -1,4 +1,5 @@ from transform import ( + build_listing_url, build_register_address, clean_listing_address, extract_full_postcode, @@ -173,3 +174,33 @@ def test_rightmove_transform_without_detail_keeps_coordinate_logic() -> None: assert result is not None assert result["Postcode"] == "SW9 7AA" assert result["Postcode source"] == "coordinates" + + +def test_build_listing_url_stamps_channel_from_new_build_flag() -> None: + # Resale gets RES_BUY; new builds get RES_NEW. + assert build_listing_url("/properties/200", False) == ( + "https://www.rightmove.co.uk/properties/200#/?channel=RES_BUY" + ) + assert build_listing_url("/properties/200", True) == ( + "https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW" + ) + # An existing channel/fragment on the source URL is stripped and re-stamped. + assert build_listing_url("/properties/200#/?channel=RES_BUY", True) == ( + "https://www.rightmove.co.uk/properties/200#/?channel=RES_NEW" + ) + # Missing URL stays empty. + assert build_listing_url("", True) == "" + + +def test_rightmove_transform_tags_new_builds_res_new() -> None: + # The Rightmove search response marks new-build developments with + # development=True; transform_property must stamp the listing URL RES_NEW. + new_build = {**_rightmove_prop(), "development": True} + result = transform_property(new_build, "SW9", StubPostcodeIndex("SW9 7AA")) + assert result is not None + assert result["Listing URL"].endswith("#/?channel=RES_NEW") + + # Ordinary resale (development absent/false) stays RES_BUY. + resale = transform_property(_rightmove_prop(), "SW9", StubPostcodeIndex("SW9 7AA")) + assert resale is not None + assert resale["Listing URL"].endswith("#/?channel=RES_BUY") diff --git a/finder/test_zoopla.py b/finder/test_zoopla.py index 228e21b..f7a993f 100644 --- a/finder/test_zoopla.py +++ b/finder/test_zoopla.py @@ -103,7 +103,7 @@ def test_parse_detail_geo_merges_location_uprn_with_address_full_address() -> No def test_parse_detail_geo_does_not_borrow_comparable_full_address() -> None: # The only `address` twin on the page belongs to a different uprn (a # comparable listing). With a uprn to match on, an unrelated twin is never - # borrowed — full_address stays None rather than grabbing the wrong street. + # borrowed: full_address stays None rather than grabbing the wrong street. html = ( '"location":{"outcode":"NR29",' '"coordinates":{"latitude":52.716014,"longitude":1.614495},' @@ -185,7 +185,7 @@ def test_parse_detail_geo_returns_none_for_garbage() -> None: assert parse_detail_geo("no data here") is None assert parse_detail_geo("") is None # Coordinates that are not inside a property location/address wrapper (e.g. - # only an unwrapped POI) yield nothing — safe degradation to the outcode. + # only an unwrapped POI) yield nothing, safe degradation to the outcode. assert parse_detail_geo('"name":"X","coordinates":{"latitude":51.5,"longitude":-0.1}') is None diff --git a/finder/transform.py b/finder/transform.py index 49986ba..2a05dd4 100644 --- a/finder/transform.py +++ b/finder/transform.py @@ -149,7 +149,7 @@ def map_property_type(sub_type: str | None) -> str: return "Terraced" if "house" in lower or "cottage" in lower: return "Detached" - log.warning("Unknown propertySubType: %r — mapping to Other", sub_type) + log.warning("Unknown propertySubType: %r, mapping to Other", sub_type) return "Other" @@ -267,7 +267,7 @@ def build_register_address( property's own number or name (e.g. Zoopla detail pages expose ``propertyNumberOrName`` = "12" or "Martham Mill"), prepend it so the address carries the house identifier that the EPC/Price-Paid register addresses also - use — turning a fuzzy street match into a near-exact one. Falls back to the + use, turning a fuzzy street match into a near-exact one. Falls back to the plain cleaned address when no number/name is available. """ cleaned = clean_listing_address(raw_address) @@ -282,6 +282,23 @@ def build_register_address( return f"{number_or_name}, {cleaned}" if cleaned else number_or_name +def build_listing_url(property_url: str | None, is_new_build: bool) -> str: + """Build the canonical Rightmove listing URL with an explicit channel marker. + + The search API always echoes ``?channel=RES_BUY`` in ``propertyUrl`` even for + new-build developments (the request channel stays BUY), so the channel is + re-stamped here from the per-listing ``development`` flag: ``RES_NEW`` for new + builds, ``RES_BUY`` for ordinary resale. The map UI reads this marker to split + new vs non-new listings. Any channel/fragment already on ``propertyUrl`` is + stripped first so the result is deterministic. + """ + if not property_url: + return "" + base = property_url.split("#", 1)[0].split("?", 1)[0] + channel = "RES_NEW" if is_new_build else "RES_BUY" + return f"{RIGHTMOVE_BASE}{base}#/?channel={channel}" + + def transform_property( prop: dict, outcode: str, @@ -337,7 +354,7 @@ def transform_property( inferred_postcode = pc_index.nearest(lat, lng) if not inferred_postcode: - log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng) + log.debug("No England postcode for property at %.4f, %.4f; skipping", lat, lng) return None raw_address = prop.get("displayAddress", "") or "" extracted_postcode = extract_full_postcode(raw_address) @@ -391,7 +408,7 @@ def transform_property( "price_frequency": "", "Price qualifier": price_qualifier, "Total floor area (sqm)": parse_display_size(prop.get("displaySize")), - "Listing URL": RIGHTMOVE_BASE + property_url if property_url else "", + "Listing URL": build_listing_url(property_url, bool(prop.get("development"))), "Listing features": key_features, "first_visible_date": prop.get("firstVisibleDate", ""), } diff --git a/finder/zoopla_flaresolverr.py b/finder/zoopla_flaresolverr.py index f3e6860..c268d83 100644 --- a/finder/zoopla_flaresolverr.py +++ b/finder/zoopla_flaresolverr.py @@ -1,7 +1,7 @@ """Zoopla scraping via FlareSolverr (no browser/VNC needed). FlareSolverr solves Zoopla's Cloudflare and returns the rendered HTML, which -still contains the React Server Components flight stream — so the existing pure +still contains the React Server Components flight stream, so the existing pure parsers work unchanged: - the search page yields the outcode's listing detail URLs, and - each detail page's flight stream carries the property's location object diff --git a/frontend/src/components/home/ProductShowcase.tsx b/frontend/src/components/home/ProductShowcase.tsx index 4273c72..5bfe64a 100644 --- a/frontend/src/components/home/ProductShowcase.tsx +++ b/frontend/src/components/home/ProductShowcase.tsx @@ -12,6 +12,7 @@ import type { TFunction } from 'i18next'; import { useTranslation } from 'react-i18next'; import { cellToLatLng, polygonToCells } from 'h3-js'; import PriceHistoryChart from '../map/PriceHistoryChart'; +import { MapErrorBoundary } from '../map/MapErrorBoundary'; import StackedBarChart from '../map/StackedBarChart'; import JourneyInstructions, { type JourneyInstructionPreset } from '../map/JourneyInstructions'; import { DualHistogram } from '../map/DualHistogram'; @@ -214,10 +215,13 @@ const SHOWCASE_MAP_START_VIEW: ViewState = { bearing: 0, }; +// End the Match fly-in over Greater London so the "match" step lands on the same +// place the Inspect (SW5 9AA) and Scout (SW5/SE22/N4) steps then drill into. The +// single-search narrative used to break by zooming to Birmingham then showing London. const SHOWCASE_MAP_END_VIEW: ViewState = { - longitude: -1.89, - latitude: 52.49, - zoom: 8.72, + longitude: -0.12, + latitude: 51.51, + zoom: 9.05, pitch: 0, bearing: 0, }; @@ -260,7 +264,12 @@ function buildShowcaseMapData(): HexagonData[] { } const SHOWCASE_MAP_DATA = buildShowcaseMapData(); -const SHOWCASE_MAP_TOTAL_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => sum + item.count, 0); +// Count only the hexes around Greater London so the Match card's figure matches the +// area it has zoomed to, instead of mislabelling the England-wide total as one city. +const SHOWCASE_LONDON_COUNT = SHOWCASE_MAP_DATA.reduce((sum, item) => { + const nearLondon = Math.abs(item.lat - 51.5072) < 0.55 && Math.abs(item.lon + 0.1276) < 0.85; + return nearLondon ? sum + item.count : sum; +}, 0); const EMPTY_SHOWCASE_POSTCODES: PostcodeFeature[] = []; const EMPTY_SHOWCASE_POIS: POI[] = []; @@ -601,37 +610,42 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) { return (
{shouldRenderMap && ( - + // The full deck.gl/maplibre map sits above the fold; contain any WebGL + // context-loss crash to this pane (and auto-recover) instead of letting it + // bubble to the top-level boundary and blank the hero. + + + )}
-
Birmingham
+
{t('home.showcaseStep2Region')}
{t('home.showcaseMatchingHomes', { - value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(), + value: SHOWCASE_LONDON_COUNT.toLocaleString(), })}
@@ -789,19 +803,22 @@ function ScoutScreen({ isActive }: { isActive: boolean }) { postcode: 'SW5 9AA', score: '94%', commute: t('home.showcaseMinutes', { count: 23 }), - price: '£492k', + perSqm: '£8,200', + delta: '−16%', }, { postcode: 'SE22 8EF', score: '91%', commute: t('home.showcaseMinutes', { count: 28 }), - price: '£518k', + perSqm: '£5,900', + delta: '−24%', }, { postcode: 'N4 2AB', score: '88%', commute: t('home.showcaseMinutes', { count: 31 }), - price: '£476k', + perSqm: '£6,400', + delta: '−21%', }, ]; @@ -934,7 +951,12 @@ function ScoutScreen({ isActive }: { isActive: boolean }) {
{row.commute}
-
{row.price}
+
+
{row.perSqm}
+
+ {row.delta} +
+
))}
@@ -1031,6 +1053,7 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps const [isStagePaused, setIsStagePaused] = useState(false); const [hasStarted, setHasStarted] = useState(false); const [canPauseOnHover, setCanPauseOnHover] = useState(false); + const [prefersReducedMotion, setPrefersReducedMotion] = useState(false); const showcaseRef = useRef(null); const inspectUserScrolledRef = useRef(false); @@ -1101,6 +1124,27 @@ export default function ProductShowcase({ className = '' }: ProductShowcaseProps return () => mediaQuery.removeEventListener('change', updateCanPause); }, []); + useEffect(() => { + if (typeof window.matchMedia !== 'function') return; + const mediaQuery = window.matchMedia('(prefers-reduced-motion: reduce)'); + const update = () => setPrefersReducedMotion(mediaQuery.matches); + update(); + mediaQuery.addEventListener('change', update); + return () => mediaQuery.removeEventListener('change', update); + }, []); + + // Under prefers-reduced-motion the progress-bar animation is disabled in CSS, so + // its onAnimationEnd (which normally advances the carousel) never fires and the + // demo would freeze on step 1. Drive the advance from a timer in that case. + useEffect(() => { + if (!prefersReducedMotion || !isProgressRunning) return; + const timer = window.setTimeout( + () => setActiveStep((step) => (step + 1) % SHOWCASE_STEP_COUNT), + activeStepIntervalMs + ); + return () => window.clearTimeout(timer); + }, [prefersReducedMotion, isProgressRunning, activeStep, activeStepIntervalMs]); + const pauseForHover = () => { if (canPauseOnHover) setIsStagePaused(true); }; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0db2755..b476a72 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -235,6 +235,7 @@ export interface HistoricalPrice { year: number; month: number; price: number; + is_new: boolean; } export interface Property { @@ -386,7 +387,7 @@ export interface HexagonStatsResponse { * sector, shown alongside the national average for each crime metric. */ crime_area_averages?: CrimeAreaAverage[]; /** Total individual crime records (last 7 years) across the selection's - * postcodes — the count behind the "individual crimes" list. */ + * postcodes, the count behind the "individual crimes" list. */ crime_total_records?: number; central_postcode?: string; /** Total usual residents (ONS Census 2021) across the postcodes in this diff --git a/pipeline/download/median_age.py b/pipeline/download/median_age.py index 922caff..23f866c 100644 --- a/pipeline/download/median_age.py +++ b/pipeline/download/median_age.py @@ -3,7 +3,7 @@ Downloads five-year age band counts (TS007A) from the NOMIS API, then computes the median age per LSOA using linear interpolation within the median class. -Source: NOMIS (ONS Census 2021 — TS007A dataset, NM_2020_1) +Source: NOMIS (ONS Census 2021, TS007A dataset, NM_2020_1) License: Open Government Licence v3.0 """ @@ -20,7 +20,7 @@ from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE" # Five-year age bands in order, with lower bounds for interpolation. -# The last band (85+) is open-ended — we treat it as 85-89 for median purposes. +# The last band (85+) is open-ended. We treat it as 85-89 for median purposes. AGE_BANDS = [ (0, 5), # Aged 0 to 4 years (5, 5), # Aged 5 to 9 years diff --git a/pipeline/download/naptan.py b/pipeline/download/naptan.py index 5a21aab..10054ed 100644 --- a/pipeline/download/naptan.py +++ b/pipeline/download/naptan.py @@ -28,7 +28,7 @@ LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU" # The Docklands Light Railway uses the analogous "ZZDL" system code (e.g. # "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a # TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family -# to its own "DLR station" category — restoring DLR to the train/tube station +# to its own "DLR station" category, restoring DLR to the train/tube station # list and giving it the DLR roundel rather than a generic tram icon. LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL" @@ -337,7 +337,7 @@ class StationAccumulator: # A single node is never both (ZZLU vs ZZDL), but a co-located # interchange (Bank, Stratford, Canning Town, West Ham) merges its LU # and DLR halves into one group carrying both flags; Tube is checked - # first so these resolve to "Tube station" — their primary identity — + # first so these resolve to "Tube station" (their primary identity), # leaving "DLR station" for the DLR-only stops the fix targets. if self.category == TRAM_METRO_CATEGORY and self.is_lu: return TUBE_STATION_CATEGORY diff --git a/pipeline/download/noise.py b/pipeline/download/noise.py index 55e7c27..c065951 100644 --- a/pipeline/download/noise.py +++ b/pipeline/download/noise.py @@ -85,24 +85,24 @@ RESOLUTION = NATIVE_RESOLUTION # Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is # otherwise inside the raster means "modelled below the lowest reporting band", -# i.e. genuinely quiet — see noise_overlay_tiles.py:167. +# i.e. genuinely quiet (see noise_overlay_tiles.py:167). NOISE_NODATA_SENTINEL = np.float32(-96.0) # Lowest modelled Defra Lden reporting band (dB). Verified against the actual # rasters: the minimum positive in-coverage value is 40.0 dB with NO values in -# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor +# (0, 40). Below the band, cells are encoded as 0.0 (genuinely quiet). We floor # in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet" # (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the # ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45. # NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles. -# NOISE_COLOR_STOPS[0]) — a rendering threshold, not the data's reporting floor. +# NOISE_COLOR_STOPS[0]), a rendering threshold, not the data's reporting floor. NOISE_QUIET_FLOOR_DB = np.float32(40.0) # Sample noise at the postcode representative point itself (no neighbourhood # window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha # of every postcode; because Defra road contours hug every modelled road and # representative points sit on/near streets, that inflated postcode noise by -# roughly +9 dB (log scale) — making ~94% of England read >=55 dB Lden and +# roughly +9 dB (log scale), making ~94% of England read >=55 dB Lden and # collapsing the metric's discrimination at the quiet end. Radius 0 -> # filter_size 1 -> the maximum_filter is skipped and each postcode reads the # 10m cell it actually sits in. @@ -110,8 +110,8 @@ POSTCODE_NOISE_RADIUS_M = 0 # Adjacent download tiles overlap by the sampling radius so every postcode's # sampling footprint is fully contained in at least one tile. With point -# sampling (radius 0) this is 0 — a representative point falls inside exactly -# one tile — but the relationship is kept so any future non-zero radius keeps +# sampling (radius 0) this is 0 (a representative point falls inside exactly +# one tile), but the relationship is kept so any future non-zero radius keeps # its window seam-safe. TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M @@ -273,7 +273,7 @@ def _download_tile( NoGeoTiffError, httpx.HTTPStatusError, # TransportError is the superset of TimeoutException, ConnectError, - # ReadError and ProtocolError — including RemoteProtocolError, raised + # ReadError and ProtocolError, including RemoteProtocolError, raised # when the WCS server closes the connection mid-stream ("incomplete # chunked read"). All are transient; retry/split rather than letting # one flaky tile crash the whole raster download. @@ -446,7 +446,7 @@ def sample_noise_at_postcodes( # Defra rasters encode TRUE nodata as the -96.0 sentinel (and # occasionally non-finite / dataset.nodata); genuinely quiet ground # below the model's lowest reporting band is encoded as 0.0. Only - # the former is "we don't know" — the latter is a real "we know it's + # the former is "we don't know". The latter is a real "we know it's # quiet" reading and must not collapse to null. So treat ONLY true # nodata as -inf (it never wins a max and never counts as coverage), # and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a @@ -555,7 +555,7 @@ def main() -> None: if not tile_paths: print( - f"[{label}] WARNING: No tiles downloaded — column will be all null" + f"[{label}] WARNING: No tiles downloaded; column will be all null" ) series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32) else: diff --git a/pipeline/download/os_greenspace.py b/pipeline/download/os_greenspace.py index 59e2768..d64ed65 100644 --- a/pipeline/download/os_greenspace.py +++ b/pipeline/download/os_greenspace.py @@ -7,7 +7,7 @@ site id and the site's polygon centroid. Sites without access points fall back to polygon centroids. Using access points rather than polygon centroids gives much more accurate -distance calculations — a property next to Hyde Park won't show 400m just +distance calculations: a property next to Hyde Park won't show 400m just because the centroid is in the middle of the park. The site id / centroid columns let downstream consumers (poi_proximity) collapse the frame back to one row per SITE for counting, so a park with 30 gates counts as one park. @@ -190,7 +190,7 @@ def download_greenspace(output: Path) -> None: print(f"Reading {site_shps[0].name} for function types...") site_funcs = _read_site_functions(site_shps[0]) - # Step 2: Read access points (primary — park entrances) + # Step 2: Read access points (primary: park entrances) print(f"Reading {access_shps[0].name}...") ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points( access_shps[0], site_funcs diff --git a/pipeline/download/places.py b/pipeline/download/places.py index 2c547a5..03a0e53 100644 --- a/pipeline/download/places.py +++ b/pipeline/download/places.py @@ -930,7 +930,7 @@ def main() -> None: df.write_parquet(args.output) print(f"Saved to {args.output}") else: - print("No places found — skipping output") + print("No places found, skipping output") if __name__ == "__main__": diff --git a/pipeline/download/rental_prices.py b/pipeline/download/rental_prices.py index 739bded..e200141 100644 --- a/pipeline/download/rental_prices.py +++ b/pipeline/download/rental_prices.py @@ -35,7 +35,7 @@ def _data_rows(df: pl.DataFrame) -> pl.DataFrame: The preamble length varies (title, optional "This worksheet contains..." note, then the header row starting with "Time period"), so locate the - header by content instead of counting rows — a fixed slice leaves the + header by content instead of counting rows: a fixed slice leaves the header in the data whenever ONS adds or removes a note line. """ header_marker = ( @@ -78,7 +78,7 @@ def _latest_rents_long(df: pl.DataFrame) -> pl.DataFrame: print(f"LAs in latest month: {df.height}") # Melt to long format: one row per area x bedroom count. - # PIPR has no Studio category — one-bed rent used as proxy for bedrooms=0. + # PIPR has no Studio category: one-bed rent used as proxy for bedrooms=0. frames = [] for col, bedrooms in [ ("rent_1bed", 0), # Studio (proxy) diff --git a/pipeline/download/tenure.py b/pipeline/download/tenure.py index 73da82f..49b999f 100644 --- a/pipeline/download/tenure.py +++ b/pipeline/download/tenure.py @@ -2,8 +2,8 @@ Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9) from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf -categories into our 3 output buckets — Owner occupied / Social rent / -Private rent — emitting one row per LSOA with the percentage of households in +categories into our 3 output buckets (Owner occupied / Social rent / +Private rent), emitting one row per LSOA with the percentage of households in each. The three buckets sum to 100%, so downstream they render as a composition/ratio (like the ethnicity and qualifications stacked bars) AND each percentage is independently filterable. @@ -19,7 +19,7 @@ NOTE this table counts HOUSEHOLDS (not usual residents). The join key downstream (merge.py) is `lsoa21`, the same key used for ethnicity, qualifications, median age, and IoD. -Source: NOMIS (ONS Census 2021 — TS054 dataset, NM_2072_1) +Source: NOMIS (ONS Census 2021, TS054 dataset, NM_2072_1) License: Open Government Licence v3.0 """ diff --git a/pipeline/download/test_ethnicity.py b/pipeline/download/test_ethnicity.py index 63f913f..5dcc0e2 100644 --- a/pipeline/download/test_ethnicity.py +++ b/pipeline/download/test_ethnicity.py @@ -72,7 +72,7 @@ def test_ethnicity_routes_chinese_to_east_and_other_asian_to_se(): def test_ethnicity_percentages_independent_per_lsoa(): - """Two LSOAs get independent profiles — the LSOA granularity is the point.""" + """Two LSOAs get independent profiles: the LSOA granularity is the point.""" df = pl.concat( [ pl.DataFrame( diff --git a/pipeline/download/test_noise.py b/pipeline/download/test_noise.py index d075590..f9e11b7 100644 --- a/pipeline/download/test_noise.py +++ b/pipeline/download/test_noise.py @@ -281,7 +281,7 @@ def test_sample_noise_preserves_genuine_reading_above_quiet_floor( # The lowest Defra reporting band is 40.0 dB; genuine readings populate # [40, ~80]. A genuine in-coverage reading at or just above the floor must be - # PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A + # PRESERVED, not clamped UP to the floor. Only true-quiet 0.0 is floored. A # quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99 # dB readings; this pins that they survive unchanged. floor = float(noise.NOISE_QUIET_FLOOR_DB) diff --git a/pipeline/download/transit_network.py b/pipeline/download/transit_network.py index fa858b7..f373beb 100644 --- a/pipeline/download/transit_network.py +++ b/pipeline/download/transit_network.py @@ -50,7 +50,7 @@ ENGLAND_PBF_URL = ( "https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf" ) -# Bus Open Data Service — pre-converted GTFS covering all England bus/tram/ferry +# Bus Open Data Service: pre-converted GTFS covering all England bus/tram/ferry BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/" # National Rail Open Data API @@ -597,7 +597,7 @@ def validate_gtfs_feed( fail("has neither calendar.txt nor calendar_dates.txt") if not _calendar_active_in_window(z, names, window_start, window_end): fail( - f"no service active between {window_start} and {window_end} — " + f"no service active between {window_start} and {window_end}: " "the feed's calendars are stale/expired and it would contribute " "zero service to routing" ) @@ -697,7 +697,7 @@ def _iter_stop_time_trips(lines, trip_id_idx: int): dtd2mysql currently writes rows grouped by trip and ordered by stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a trip_id reappearing later raises instead of silently scrambling trips); - within-trip order is NOT assumed — callers sort each group by its original + within-trip order is NOT assumed: callers sort each group by its original stop_sequence. """ current_trip: str | None = None @@ -1175,7 +1175,7 @@ def main() -> None: cif = download_national_rail_cif(raw_dir) if cif is None: raise RuntimeError( - "National Rail timetable was not downloaded — set " + "National Rail timetable was not downloaded: set " "NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at " "https://opendata.nationalrail.co.uk/). National Rail heavy rail is " "required; without it the transit network models every train journey " diff --git a/pipeline/transform/postcode_boundaries/geometry.py b/pipeline/transform/postcode_boundaries/geometry.py index 3fd85a4..4172f48 100644 --- a/pipeline/transform/postcode_boundaries/geometry.py +++ b/pipeline/transform/postcode_boundaries/geometry.py @@ -1,8 +1,8 @@ """Robust GEOS overlay helpers. Overlay operations (union, difference, intersection) can raise a -``GEOSException`` — most often ``TopologyException: side location conflict``, -``Ring edge missing``, or ``found non-noded intersection`` — on geometries that +``GEOSException``, most often ``TopologyException: side location conflict``, +``Ring edge missing``, or ``found non-noded intersection``, on geometries that contain near-coincident or near-degenerate edges, or that are individually invalid. The robust remedy is a *fixed-precision* overlay: GEOS's OverlayNG engine, handed a grid size, nodes every edge onto that grid and finishes where @@ -14,8 +14,8 @@ learned the hard way from a crash: 1. **Never precision-reduce with the default mode.** ``set_precision``'s default ``valid_output`` (and ``keep_collapsed``) mode runs its *own* noding pass that re-raises the very ``side location conflict`` we are trying to escape. We push - the grid into the overlay via the ``grid_size`` argument instead — where - OverlayNG nodes robustly — and only ever call ``set_precision`` in + the grid into the overlay via the ``grid_size`` argument instead (where + OverlayNG nodes robustly) and only ever call ``set_precision`` in ``pointwise`` mode (pure coordinate rounding, which cannot raise). 2. **Validate first.** ``make_valid`` repairs the self-intersections (bow-ties, pinches) that make GEOS choke, so the overlay starts from an OGC-valid shape. @@ -40,7 +40,7 @@ from shapely import GEOSException, make_valid, set_precision from shapely.geometry import Polygon from shapely.ops import unary_union -# 0.1 mm in metres — well below MIN_GEOM_AREA (0.01 m^2) and survey resolution. +# 0.1 mm in metres: well below MIN_GEOM_AREA (0.01 m^2) and survey resolution. _SNAP_GRID = 1e-4 _EMPTY = Polygon() diff --git a/pipeline/transform/postcode_boundaries/output.py b/pipeline/transform/postcode_boundaries/output.py index 4d5aa58..0356459 100644 --- a/pipeline/transform/postcode_boundaries/output.py +++ b/pipeline/transform/postcode_boundaries/output.py @@ -120,7 +120,7 @@ def _is_pointlike(geom_bng) -> bool: def _rescue_footprint(geom_bng) -> dict | None: """Fatten a degenerate BNG geometry into a representable footprint and snap. - A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon — the + A POINTLIKE input (a point, or a near-zero-area/short-perimeter polygon, the signature of a tower-block postcode whose UPRNs all share one coordinate) gets a building-scale buffer so it is not reduced to an invisible sub-metre dot; thin slivers that still carry length keep the minimal buffer. @@ -263,7 +263,7 @@ def merge_fragments( # Close tiny gaps between adjacent OA boundary edges (float mismatches). # The closing can erode a tiny MultiPolygon (e.g. a postcode with only a # sliver fragment) to nothing, which would leave the postcode with no - # geometry at all — keep the un-closed shape if that happens. + # geometry at all. Keep the un-closed shape if that happens. if combined.geom_type == "MultiPolygon": closed = combined.buffer(5.0).buffer(-5.0) if not closed.is_valid: @@ -308,7 +308,7 @@ def _polygonal(geom): return None # Both callers run on WGS84-degree output geometry, so the robustness # fallback snaps on the 1e-6° grid (~0.11 m), not geometry.py's metre - # default — a coarse metre grid would obliterate a degree-scale shape. + # default. A coarse metre grid would obliterate a degree-scale shape. merged = safe_union(polys, grid=_OUTPUT_PRECISION_DEG) return merged if not merged.is_empty else None return None @@ -324,7 +324,7 @@ def _resolve_overlaps( containment (a postcode fully enclosed by another). Each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area**: a smaller postcode wins contested ground. That single rule - handles both cases correctly — an enclosed postcode is always smaller than its + handles both cases correctly: an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (a `overlaps` query alone would miss containment entirely). Run last, on the final output geometries, so nothing re-introduces overlap afterwards. A postcode that would @@ -348,7 +348,7 @@ def _resolve_overlaps( arr = np.array(geoms, dtype=object) pairs: set[tuple[int, int]] = set() # "overlaps" gives partial overlaps; "contains" gives containment (which - # "overlaps" excludes) — together they cover every 2-D overlap without the + # "overlaps" excludes). Together they cover every 2-D overlap without the # edge-touch explosion a plain "intersects" query would add. for predicate in ("overlaps", "contains"): qsrc, qtgt = tree.query(arr, predicate=predicate) @@ -577,7 +577,7 @@ def _grid_footprint(geom): pass can shave a small (e.g. co-located, non-geographic) postcode down to a sub-grid sliver that disappears when snapped to output precision. Rather than drop it, place a minimal valid footprint at its location. The tiny overlap - this re-creates with the neighbour that trimmed it is harmless — the output + this re-creates with the neighbour that trimmed it is harmless: the output partition is best-effort, a missing boundary is a hard validation failure. """ try: diff --git a/pipeline/transform/postcode_boundaries/process_oa.py b/pipeline/transform/postcode_boundaries/process_oa.py index 4404560..50555d6 100644 --- a/pipeline/transform/postcode_boundaries/process_oa.py +++ b/pipeline/transform/postcode_boundaries/process_oa.py @@ -11,7 +11,7 @@ from .voronoi import compute_voronoi_regions MIN_GEOM_AREA = 0.01 # Minimal footprint (BNG metres) for a postcode whose UPRN seed wins no area in a -# crowded multi-postcode OA — its Voronoi cell ∩ remaining collapses below +# crowded multi-postcode OA: its Voronoi cell ∩ remaining collapses below # MIN_GEOM_AREA, or its seed sits inside an INSPIRE parcel wholly claimed by a # co-located postcode. Every *active* postcode must keep a boundary # (validate_outputs is zero-tolerance), so it gets a small disc at its true seed @@ -77,7 +77,7 @@ def process_oa( fragments.append((pc, merged)) # Every postcode with a UPRN seed in this OA must keep at least a minimal - # footprint — in a dense OA (a block of flats with hundreds of distinct + # footprint: in a dense OA (a block of flats with hundreds of distinct # postcodes) a single-seed postcode's cell can collapse below MIN_GEOM_AREA or # be fully absorbed by a co-located postcode's INSPIRE parcel, producing no # fragment, and an active postcode must never be dropped. @@ -142,7 +142,7 @@ def _claim_inspire_parcels( # UPRNs from a single postcode goes wholly to that postcode. A parcel shared # by several postcodes (a block of flats spanning postcodes, or overlapping # parcel data) is split between them via a sub-Voronoi over their own UPRNs - # clipped to the parcel — so EVERY contained postcode keeps part of the + # clipped to the parcel, so EVERY contained postcode keeps part of the # parcel. A bare majority vote would hand the whole parcel to one winner and # leave the losers' UPRNs trapped inside claimed land, dropping them from # both this claim and the `remaining` polygon handed to Voronoi downstream. @@ -312,7 +312,7 @@ def _extract_polygonal(geom) -> Polygon | MultiPolygon | None: return polys[0] # Union (not bare MultiPolygon construction): make_valid can emit # overlapping polygonal parts, and a MultiPolygon of overlapping parts is - # invalid — it double-counts area and makes the next `.difference()` raise + # invalid: it double-counts area and makes the next `.difference()` raise # a TopologyException that aborts the OA (and, in parallel mode, the # worker). safe_union merges them into a valid geometry. merged = safe_union(polys) diff --git a/pipeline/transform/postcode_boundaries/uprn.py b/pipeline/transform/postcode_boundaries/uprn.py index d7d19b8..6c99e94 100644 --- a/pipeline/transform/postcode_boundaries/uprn.py +++ b/pipeline/transform/postcode_boundaries/uprn.py @@ -82,7 +82,7 @@ def load_uprns( # Remap terminated postcodes to their nearest active successor. The # successor generally lives in a DIFFERENT OA (and at different grid # coordinates), so the remapped point must adopt the successor's - # authoritative OA/coords — keeping the terminated postcode's original + # authoritative OA/coords. Keeping the terminated postcode's original # OA would seed the successor into an OA it doesn't belong to, splitting # its boundary across OAs. Genuine (non-remapped) UPRN rows keep their # own OA, since a live postcode can legitimately span several OAs. @@ -127,7 +127,7 @@ def load_uprns( uprns.sort("OA21CD").sink_parquet(tmp_path) release_memory() - # Read the sorted data — only one copy in memory (~2GB) + # Read the sorted data: only one copy in memory (~2GB) df = pl.read_parquet(tmp_path) tmp_path.unlink() n = len(df) diff --git a/pipeline/transform/price_estimation/index.py b/pipeline/transform/price_estimation/index.py index 0994939..65230e9 100644 --- a/pipeline/transform/price_estimation/index.py +++ b/pipeline/transform/price_estimation/index.py @@ -4,7 +4,7 @@ Stratified by property type and postcode sector, with IRLS Huber regression, hierarchical shrinkage (sector → district → area → national → hedonic), and KD-tree spatial smoothing for sparse sectors. -Output: price_index.parquet — sector x type_group x year -> log_index +Output: price_index.parquet (sector x type_group x year -> log_index) """ import argparse diff --git a/pipeline/transform/price_estimation/utils.py b/pipeline/transform/price_estimation/utils.py index e551e17..e0e73ba 100644 --- a/pipeline/transform/price_estimation/utils.py +++ b/pipeline/transform/price_estimation/utils.py @@ -23,7 +23,7 @@ ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"] # Natural join key from estimates back onto properties: postcode plus the # coalesced register/EPC address. This is unique and non-null on the deduped # dwelling universe (see property_base._dedupe_collapsed_properties), so it maps -# estimates 1:1 onto properties regardless of row order — estimates are computed +# estimates 1:1 onto properties regardless of row order. Estimates are computed # from a separate price_inputs.parquet, so a positional key would not line up. JOIN_ADDRESS = "_join_address" JOIN_KEYS = ["Postcode", JOIN_ADDRESS] diff --git a/pipeline/transform/test_join_epc_pp.py b/pipeline/transform/test_join_epc_pp.py index 6438caf..8940495 100644 --- a/pipeline/transform/test_join_epc_pp.py +++ b/pipeline/transform/test_join_epc_pp.py @@ -315,7 +315,7 @@ def test_run_tenure_history_tracks_rent_owner_transitions(tmp_path: Path): def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path): # A property only ever observed as owner-occupied has no tenure change worth - # surfacing — the timeline column is null (no events), not a noisy baseline. + # surfacing: the timeline column is null (no events), not a noisy baseline. zip_path = tmp_path / "domestic-csv.zip" _write_epc_zip( zip_path, @@ -339,6 +339,37 @@ def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path): assert df.get_column("tenure_history").to_list() == [None] +def test_run_latest_tenure_status_reflects_most_recent_certificate(tmp_path: Path): + # Two certificates for one dwelling: an older social-rented cert and a newer + # owner-occupied one. The published latest_tenure_status must carry the + # LATEST certificate's normalized tenure ("Owner-occupied"), while + # was_council_house stays "Yes" because the dwelling was social at some + # point. This also confirms latest_tenure_status reaches the epc_pp parquet. + zip_path = tmp_path / "domestic-csv.zip" + _write_epc_zip( + zip_path, + [ + _row(inspection_date="2016-04-01", tenure="Rented (social)"), + _row(inspection_date="2024-04-01", tenure="owner-occupied"), + ], + ) + + price_paid_path = tmp_path / "price-paid.parquet" + _price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet( + price_paid_path + ) + + output_path = tmp_path / "epc-pp.parquet" + _run(zip_path, price_paid_path, output_path, tmp_path) + + df = pl.read_parquet(output_path) + + assert df.height == 1 + assert df.select("latest_tenure_status", "was_council_house").to_dicts() == [ + {"latest_tenure_status": "Owner-occupied", "was_council_house": "Yes"} + ] + + def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path): # Two certificates for the same property. The cert with the garbled, # unparseable inspection_date must NOT be chosen as "latest": a string sort @@ -565,7 +596,7 @@ def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: P price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { - # 5_000 is below MIN_PRICE (10_000) — a nominal/junk transfer that + # 5_000 is below MIN_PRICE (10_000), a nominal/junk transfer that # must still anchor the construction year but stay out of the price # aggregations. "price": [5_000, 300_000], @@ -603,7 +634,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path): # lands AFTER its first Land Registry sale (1998). A dwelling cannot have # been built after it was first sold, so the published build year must be # capped at the first transfer year (1998), not the later band estimate. - # It stays flagged as an estimate (approximate=1) — it is still EPC-derived. + # It stays flagged as an estimate (approximate=1). It is still EPC-derived. zip_path = tmp_path / "domestic-csv.zip" _write_epc_zip( zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")] @@ -627,7 +658,7 @@ def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path): def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path): # The common case: the EPC band (1950-1966 -> 1958) predates the first - # recorded sale (2020). The cap must NOT fire — the band estimate stands. + # recorded sale (2020). The cap must NOT fire. The band estimate stands. zip_path = tmp_path / "domestic-csv.zip" _write_epc_zip(zip_path) @@ -649,7 +680,7 @@ def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path): def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path): # A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the # NEW floor (10k): it must now be RETAINED in the price aggregations. This - # pins the 50k->10k change — it fails on the pre-fix 50k floor (where 30k was + # pins the 50k->10k change: it fails on the pre-fix 50k floor (where 30k was # excluded, giving historical_prices length 1 / latest_price 250_000). zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: @@ -748,8 +779,8 @@ def test_run_collapses_duplicate_transactions(tmp_path: Path): # The duplicated 250_000 sale collapses to one entry; two distinct sales. assert df.get_column("historical_prices").to_list() == [ [ - {"year": 2020, "month": 2, "price": 200_000}, - {"year": 2024, "month": 2, "price": 250_000}, + {"year": 2020, "month": 2, "price": 200_000, "is_new": False}, + {"year": 2024, "month": 2, "price": 250_000, "is_new": False}, ] ] assert df.get_column("latest_price").to_list() == [250_000] @@ -777,7 +808,7 @@ def test_run_excludes_implausible_price_jump_but_keeps_property(tmp_path: Path): assert df.height == 1 assert df.get_column("latest_price").to_list() == [140_000] assert df.get_column("historical_prices").to_list() == [ - [{"year": 2016, "month": 6, "price": 140_000}] + [{"year": 2016, "month": 6, "price": 140_000, "is_new": False}] ] diff --git a/pipeline/transform/test_transform_poi.py b/pipeline/transform/test_transform_poi.py index 11bb3a4..7b5399f 100644 --- a/pipeline/transform/test_transform_poi.py +++ b/pipeline/transform/test_transform_poi.py @@ -593,7 +593,7 @@ def test_transform_grocery_dedup_drops_only_grocery_aspect(tmp_path): # The _write_transform_inputs fixture seeds 5 GEOLYTIX "Tesco" points at # (51.52, -0.14). An OSM object colocated there carrying "Tesco" in its name # is the same physical store, so its Convenience Store (Groceries) row is a - # duplicate and must be dropped — but its NON-grocery aspect (a Post Office + # duplicate and must be dropped, but its NON-grocery aspect (a Post Office # sharing the same OSM id) must survive. An independent shop away from the # GEOLYTIX point keeps its grocery row. raw = pl.DataFrame( diff --git a/pipeline/transform/transform_poi.py b/pipeline/transform/transform_poi.py index 4fae0a9..dac448f 100644 --- a/pipeline/transform/transform_poi.py +++ b/pipeline/transform/transform_poi.py @@ -34,7 +34,7 @@ DROP_CATEGORIES = { "emergency/water_tank", "leisure/bleachers", "leisure/schoolyard", - # Park "furniture" / incidental features — not parks; they massively + # Park "furniture" / incidental features, not parks; they massively # inflated the Park count (picnic_table ~15k, outdoor_seating ~5.8k). "leisure/bandstand", "leisure/bird_hide", @@ -222,7 +222,7 @@ DROP_CATEGORIES = { "public_transport/entrance", "public_transport/station", "public_transport/stop_position", - # Education amenities — schools come from GIAS instead. OSM coverage for + # Education amenities. Schools come from GIAS instead. OSM coverage for # tertiary education, tutoring, and childcare is too noisy/incomplete to be # useful on a property-search map. "amenity/school", @@ -398,7 +398,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [ "tourism/theme_park", # bicycle_rental/boat_rental/marina/slipway used to live here and # made up ~46% of the bucket (cycle-hire docks, boat ramps); they - # are infrastructure, not entertainment venues — see DROP_CATEGORIES. + # are infrastructure, not entertainment venues. See DROP_CATEGORIES. "leisure/hackerspace", "leisure/yes", ], @@ -722,7 +722,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [ [ "leisure/fitness_centre", # leisure/fitness_station (outdoor pull-up bars / trim-trail - # apparatus, ~2.5k) is not a gym — see DROP_CATEGORIES. + # apparatus, ~2.5k) is not a gym. See DROP_CATEGORIES. "amenity/dojo", "amenity/dancing_school", ], @@ -849,8 +849,8 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [ "healthcare/pharmacy", "shop/chemist", # healthcare/alternative, shop/herbalist and shop/health (homeopaths, - # herbalists, generic "health" shops) are not dispensing pharmacies - # — see DROP_CATEGORIES. + # herbalists, generic "health" shops) are not dispensing pharmacies. + # See DROP_CATEGORIES. ], ), # "Hospital & Clinic" used to be one bucket; an actual hospital and a small @@ -878,7 +878,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [ "healthcare/laboratory", "healthcare/rehabilitation", "healthcare/vaccination_centre", - # healthcare/yes (untyped junk rows) is dropped — see DROP_CATEGORIES. + # healthcare/yes (untyped junk rows) is dropped. See DROP_CATEGORIES. ], ), ( @@ -950,7 +950,7 @@ _CATEGORIES: list[tuple[str, str, str, list[str]]] = [ [ "tourism/gallery", # tourism/artwork (statues, murals, village signs) was 93% of this - # bucket and is not a visitable gallery — see DROP_CATEGORIES. + # bucket and is not a visitable gallery. See DROP_CATEGORIES. ], ), ( @@ -1420,7 +1420,7 @@ def _school_icon_category_expr() -> pl.Expr: # GIAS phase mixes casing ("Middle deemed Primary" vs "Middle deemed # primary") so we normalise before matching. phase = pl.col("phase").str.to_lowercase() - # gias._format_age_range emits three shapes: "–" (em-dash), + # gias._format_age_range emits three shapes: "–" (en-dash), # "up to " (high-only) and "+" (low-only). Extract the leading # integer as low and the trailing integer as high, then suppress the wrong # end for the one-sided shapes so they don't collapse to a single bound. @@ -1477,7 +1477,7 @@ def _load_ofsted_ratings(ofsted_path: Path) -> pl.LazyFrame: the conventional Ofsted labels; when there is no usable graded result (null/"Not judged", e.g. schools last seen under the post-2024 ungraded report-card framework) we fall back to "Ungraded inspection overall outcome" - so genuinely good/outstanding schools aren't dropped — mirroring + so genuinely good/outstanding schools aren't dropped, mirroring school_catchments.classify_good_plus_schools. Remaining nulls drop out.""" grade_col = pl.col("Latest OEIF overall effectiveness") # See school_catchments: the ungraded outcome carries "School remains Good"/ @@ -1682,7 +1682,7 @@ def osm_groceries_colocated_with_geolytix( An OSM Groceries row is a duplicate when a GEOLYTIX point lies within ``radius_m`` metres AND that point's brand tokens (its ``category``, e.g. - "Tesco", "Co-op", "M&S") are all present in the OSM row's name — i.e. the + "Tesco", "Co-op", "M&S") are all present in the OSM row's name, i.e. the same physical branded store. Short brands like "M&S" match via _GROCERY_SHORT_BRAND_TOKENS; brands that still tokenise to set() are kept. @@ -1703,7 +1703,7 @@ def osm_groceries_colocated_with_geolytix( osm_ids = osm_groceries["id"].to_list() osm_name_tokens = [_significant_tokens(n) for n in osm_groceries["name"].to_list()] - # Equirectangular projection to metres around the shared mean latitude — at + # Equirectangular projection to metres around the shared mean latitude. At # England's scale this is accurate to well under the dedup radius. mean_lat = float(np.mean(np.concatenate([glx_lat, osm_lat]))) cos_lat = float(np.cos(np.radians(mean_lat))) @@ -1860,7 +1860,7 @@ def transform( ) # Scope the drop to the Groceries group: a single OSM object can also # carry a non-grocery aspect (e.g. a convenience store that is also a - # Post Office), which must survive — only its duplicate grocery row goes. + # Post Office), which must survive. Only its duplicate grocery row goes. lf = lf.filter( ~((pl.col("group") == "Groceries") & pl.col("id").is_in(duplicate_ids)) ) diff --git a/pipeline/transform/tree_density.py b/pipeline/transform/tree_density.py index 75ba03e..8eb7275 100644 --- a/pipeline/transform/tree_density.py +++ b/pipeline/transform/tree_density.py @@ -45,7 +45,7 @@ POSTCODE_DENSITY_PERCENTILE_COL = "Tree canopy density percentile within {radius POSTCODE_AREA_COL = "Tree canopy area within {radius}m (sqm)" POSTCODE_HEIGHT_COL = "Mean TOW height within {radius}m (m)" -# National Forest Inventory (NFI) woodland — the geometric complement of TOW. +# National Forest Inventory (NFI) woodland: the geometric complement of TOW. # NFI ships as a zipped shapefile of woodland parcels (>=0.5 ha) in EPSG:27700. # Field names are from the NFI Woodland England 2022 release; re-check on bumps. NFI_CATEGORY_COL = "CATEGORY" @@ -263,7 +263,7 @@ def _postcode_buffers( return circles, shapely.STRtree(circles) -# 0.1 mm in the BNG working CRS (EPSG:27700) — far below survey resolution; the +# 0.1 mm in the BNG working CRS (EPSG:27700), far below survey resolution; the # same grid the postcode_boundaries overlay uses. _OVERLAY_GRID_M = 1e-4 @@ -274,13 +274,13 @@ def _robust_intersection_area(a: np.ndarray, b: np.ndarray) -> np.ndarray: External Forest Research TOW/NFI polygons are occasionally invalid (self-intersections), and a single bad polygon makes the batched ``shapely.intersection`` raise ``TopologyException: side location conflict``, - aborting the whole run. The fast path is the raw batched overlay — unchanged, - full-speed, when the data is clean — and only a failure triggers repair. + aborting the whole run. The fast path is the raw batched overlay (unchanged, + full-speed, when the data is clean), and only a failure triggers repair. The repair deliberately uses a *plain* overlay rather than the fixed-precision (``grid_size``) one: ``make_valid`` can emit a mixed-dimension ``GeometryCollection`` (a polygon plus a dangling line), which OverlayNG - rejects with ``Overlay input is mixed-dimension`` — whereas a plain overlay + rejects with ``Overlay input is mixed-dimension``, whereas a plain overlay accepts it, and its non-polygonal debris has zero area and is dropped by the ``clipped_area > 0`` filter downstream anyway. A final pointwise coordinate snap (which never raises) collapses the near-coincident edges behind any @@ -496,7 +496,7 @@ def _finalize_metrics( if over_count: print( f" note: {over_count:,} postcode(s) exceeded 100% raw canopy and were " - "capped — indicates overlapping TOW/NFI canopy within the buffer" + "capped, indicating overlapping TOW/NFI canopy within the buffer" ) mean_height = np.divide( diff --git a/pipeline/utils/test_poi_counts.py b/pipeline/utils/test_poi_counts.py index 59d34c2..008c37d 100644 --- a/pipeline/utils/test_poi_counts.py +++ b/pipeline/utils/test_poi_counts.py @@ -92,7 +92,7 @@ def test_custom_radius(pois): } ) - # 0.01 km = 10m — only the POI at the exact same location should match + # 0.01 km = 10m: only the POI at the exact same location should match result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=0.01) # The Restaurant at (51.5074, -0.1278) is at distance 0 assert result["restaurants_0km"][0] >= 1 @@ -135,9 +135,9 @@ def test_min_distance_finds_nearest(postcodes, pois): assert len(result) == 2 ec1a = result.filter(pl.col("postcode") == "EC1A 1BB") - # Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278) — very close + # Rail station is at (51.5073, -0.1277), postcode at (51.5074, -0.1278), very close assert ec1a["train_tube_nearest_km"][0] < 0.05 # within 50m - # Restaurant is co-located — distance ~0 + # Restaurant is co-located: distance ~0 assert ec1a["restaurants_nearest_km"][0] < 0.01 # Far-away postcode should still get the global nearest distance.