diff --git a/.gitignore b/.gitignore index abd3d65..d07fdcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .venv .claude -tfl_journey_client **/node_modules **/__pycache__ **/dist diff --git a/Makefile.data b/Makefile.data index fcedd88..bf5f683 100644 --- a/Makefile.data +++ b/Makefile.data @@ -51,6 +51,7 @@ PLACES := $(DATA_DIR)/places.parquet LISTINGS_BUY := $(DATA_DIR)/online_listings_buy.parquet LISTINGS_RENT := $(DATA_DIR)/online_listings_rent.parquet LSOA_POP := $(DATA_DIR)/lsoa_population.parquet +RM_OUTCODES := frontend/src/lib/rightmove-outcodes.json # Sentinel files for directory targets (Make doesn't track directories well) GEOSURE_STAMP := $(GEOSURE_DIR)/.done @@ -64,7 +65,7 @@ PMTILES_VERSION := 1.22.3 download-arcgis download-price-paid download-deprivation download-ethnicity \ download-naptan download-pois download-ofsted download-broadband download-rental-prices \ download-postcodes download-geosure download-noise download-inspire \ - download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population \ + download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population download-rightmove-outcodes \ transform-pois transform-epc-pp transform-crime transform-poi-proximity \ transform-school-proximity transform-geosure transform-postcode-boundaries \ generate-postcode-boundaries @@ -92,6 +93,7 @@ download-greenspace: $(GREENSPACE) download-pbf: $(PBF) download-places: $(PLACES) download-lsoa-population: $(LSOA_POP) +download-rightmove-outcodes: $(RM_OUTCODES) transform-pois: $(POIS_FILTERED) transform-epc-pp: $(EPC_PP) transform-crime: $(CRIME) @@ -187,6 +189,9 @@ $(PLACES): $(PBF) $(LSOA_POP): uv run python -m pipeline.download.lsoa_population --output $@ +$(RM_OUTCODES): $(MERGE_STAMP) + uv run python -m pipeline.download.rightmove_outcodes --postcodes $(POSTCODES_PQ) --output $@ + # ── Transforms ──────────────────────────────────────────────────────────────── $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) diff --git a/finder/constants.py b/finder/constants.py index a1ebdcc..f7ee884 100644 --- a/finder/constants.py +++ b/finder/constants.py @@ -14,7 +14,10 @@ SEED = 42 # Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable. SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3")) # Whether to run a scrape immediately on startup -RUN_ON_STARTUP = False +RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes") +# Enable/disable individual sources +SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes") +SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes") TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead" SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search" diff --git a/finder/storage.py b/finder/storage.py index c2b9f81..deda74f 100644 --- a/finder/storage.py +++ b/finder/storage.py @@ -56,7 +56,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None: "Address per Property Register": [ p["Address per Property Register"] for p in properties ], - "Leashold/Freehold": [p["Leashold/Freehold"] for p in properties], + "Leasehold/Freehold": [p["Leasehold/Freehold"] for p in properties], "Property type": [p["Property type"] for p in properties], "Property sub-type": [p["Property sub-type"] for p in properties], "Price qualifier": [p["Price qualifier"] for p in properties], @@ -78,7 +78,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None: "lat": pl.Float64, "Postcode": pl.Utf8, "Address per Property Register": pl.Utf8, - "Leashold/Freehold": pl.Utf8, + "Leasehold/Freehold": pl.Utf8, "Property type": pl.Utf8, "Property sub-type": pl.Utf8, "Price qualifier": pl.Utf8, diff --git a/frontend/src/components/map/HoverCard.tsx b/frontend/src/components/map/HoverCard.tsx index b9a99d8..adf6e6b 100644 --- a/frontend/src/components/map/HoverCard.tsx +++ b/frontend/src/components/map/HoverCard.tsx @@ -59,16 +59,6 @@ export default memo(function HoverCard({ x, y, id, isPostcode, data, filters, fe transform: 'translate(-50%, -100%)', }} > - {/* Arrow */} -
-
{/* Header */}
diff --git a/frontend/src/components/map/Map.tsx b/frontend/src/components/map/Map.tsx index 4b110fc..9b9b687 100644 --- a/frontend/src/components/map/Map.tsx +++ b/frontend/src/components/map/Map.tsx @@ -14,7 +14,7 @@ import type { } from '../../types'; import { zoomToResolution, getBoundsFromViewState, getMapStyle } from '../../lib/map-utils'; -import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS } from '../../lib/consts'; +import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS, POI_GROUP_COLORS, POI_DEFAULT_COLOR } from '../../lib/consts'; import LocationSearch, { type SearchedLocation } from './LocationSearch'; import MapLegend from './MapLegend'; import HoverCard from './HoverCard'; @@ -263,25 +263,49 @@ export default memo(function Map({ ))} {popupInfo && (
- {popupInfo.name} -
{popupInfo.category}
- {osmIdToUrl(popupInfo.id) && ( - - View on OSM - + {popupInfo.isCluster ? ( +
+
+ {popupInfo.clusterCount} +
+
places
+
+ ) : ( +
+
+ {popupInfo.emoji} +
+
{popupInfo.name}
+
+ + {popupInfo.category} +
+
+
+ {osmIdToUrl(popupInfo.id) && ( + + View on OSM + + )} +
)}
)} diff --git a/frontend/src/components/map/PropertiesPane.tsx b/frontend/src/components/map/PropertiesPane.tsx index 4d63bdc..c97bf0b 100644 --- a/frontend/src/components/map/PropertiesPane.tsx +++ b/frontend/src/components/map/PropertiesPane.tsx @@ -135,17 +135,6 @@ function PropertyLoadingSkeleton() { ); } -const LISTING_STATUS_STYLES: Record = { - 'For sale': 'bg-teal-100 text-teal-800 dark:bg-teal-900/40 dark:text-teal-300', - 'For rent': 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300', - 'Historical sale': 'bg-warm-100 text-warm-600 dark:bg-warm-700 dark:text-warm-300', -}; - -function ListingStatusBadge({ status }: { status: string }) { - const style = LISTING_STATUS_STYLES[status] ?? LISTING_STATUS_STYLES['Historical sale']; - return {status}; -} - function PropertyCard({ property }: { property: Property }) { const price = getNum(property, 'Last known price'); const estimatedPrice = getNum(property, 'Estimated current price'); @@ -161,18 +150,13 @@ function PropertyCard({ property }: { property: Property }) { const bathrooms = getNum(property, 'Bathrooms'); const listingDate = getNum(property, 'Listing date'); - const listingStatus = property.listing_status; - return (
-
-
-
- {property.address || 'Unknown Address'} -
-
{property.postcode}
+
+
+ {property.address || 'Unknown Address'}
- {listingStatus && } +
{property.postcode}
{property.property_sub_type && ( diff --git a/frontend/src/hooks/useDeckLayers.ts b/frontend/src/hooks/useDeckLayers.ts index 275eb0f..26e8462 100644 --- a/frontend/src/hooks/useDeckLayers.ts +++ b/frontend/src/hooks/useDeckLayers.ts @@ -1,7 +1,8 @@ import { useCallback, useRef, useState, useMemo, useEffect } from 'react'; import { H3HexagonLayer } from '@deck.gl/geo-layers'; -import { GeoJsonLayer, IconLayer, TextLayer } from '@deck.gl/layers'; +import { GeoJsonLayer, IconLayer, TextLayer, ScatterplotLayer } from '@deck.gl/layers'; import { cellToBoundary } from 'h3-js'; +import Supercluster from 'supercluster'; import type { PickingInfo } from '@deck.gl/core'; import type { HexagonData, @@ -12,7 +13,16 @@ import type { FeatureMeta, Bounds, } from '../types'; -import { DENSITY_GRADIENT, DENSITY_GRADIENT_DARK } from '../lib/consts'; +import { + DENSITY_GRADIENT, + DENSITY_GRADIENT_DARK, + POI_GROUP_COLORS, + POI_DEFAULT_COLOR, + MINOR_POI_CATEGORIES, + MINOR_POI_ZOOM_THRESHOLD, + POI_CLUSTER_RADIUS, + POI_CLUSTER_MAX_ZOOM, +} from '../lib/consts'; import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils'; import { type TravelTimeEntry, @@ -55,7 +65,18 @@ interface PopupInfo { y: number; name: string; category: string; + group: string; + emoji: string; id: string; + isCluster?: boolean; + clusterCount?: number; +} + +interface ClusterPoint { + lng: number; + lat: number; + count: number; + clusterId: number; } export function useDeckLayers({ @@ -204,6 +225,8 @@ export function useDeckLayers({ y: info.y, name: info.object.name, category: info.object.category, + group: info.object.group, + emoji: info.object.emoji, id: info.object.id, }); } else { @@ -217,6 +240,30 @@ export function useDeckLayers({ handlePoiHoverRef.current(info); }, []); + const handleClusterHover = useCallback((info: PickingInfo) => { + if (info.object && info.x !== undefined && info.y !== undefined) { + setPopupInfo({ + x: info.x, + y: info.y, + name: `${info.object.count} places`, + category: 'Zoom in to see details', + group: '', + emoji: '', + id: '', + isCluster: true, + clusterCount: info.object.count, + }); + } else { + setPopupInfo(null); + } + }, []); + + const handleClusterHoverRef = useRef(handleClusterHover); + handleClusterHoverRef.current = handleClusterHover; + const stableClusterHover = useCallback((info: PickingInfo) => { + handleClusterHoverRef.current(info); + }, []); + // eslint-disable-next-line @typescript-eslint/no-explicit-any const handlePostcodeClick = useCallback((info: PickingInfo) => { const pc = info.object?.properties?.postcode; @@ -461,24 +508,147 @@ export function useDeckLayers({ [postcodeData, theme] ); - const poiLayer = useMemo( + // --- POI clustering --- + const clusterIndex = useMemo(() => { + if (pois.length === 0) return null; + const index = new Supercluster({ + radius: POI_CLUSTER_RADIUS, + maxZoom: POI_CLUSTER_MAX_ZOOM, + }); + const features: Supercluster.PointFeature[] = pois.map((poi) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [poi.lng, poi.lat] }, + properties: poi, + })); + index.load(features); + return index; + }, [pois]); + + const clusterZoom = Math.floor(zoom); + const showMinorPois = zoom >= MINOR_POI_ZOOM_THRESHOLD; + + const { visiblePois, clusters } = useMemo(() => { + if (!clusterIndex || pois.length === 0) { + return { visiblePois: [] as POI[], clusters: [] as ClusterPoint[] }; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const allFeatures = clusterIndex.getClusters([-180, -85, 180, 85], clusterZoom) as any[]; + const individual: POI[] = []; + const clusterPoints: ClusterPoint[] = []; + for (const feature of allFeatures) { + if (feature.properties.cluster) { + clusterPoints.push({ + lng: feature.geometry.coordinates[0], + lat: feature.geometry.coordinates[1], + count: feature.properties.point_count, + clusterId: feature.properties.cluster_id, + }); + } else { + const poi = feature.properties as POI; + if (!showMinorPois && MINOR_POI_CATEGORIES.has(poi.category)) continue; + individual.push(poi); + } + } + return { visiblePois: individual, clusters: clusterPoints }; + }, [clusterIndex, clusterZoom, showMinorPois, pois]); + + // --- Individual POI layers (shadow → background → emoji) --- + const poiShadowLayer = useMemo( + () => + new ScatterplotLayer({ + id: 'poi-shadow', + data: visiblePois, + getPosition: (d) => [d.lng, d.lat], + getRadius: 16, + radiusUnits: 'pixels', + getFillColor: isDark ? [0, 0, 0, 50] : [0, 0, 0, 25], + pickable: false, + transitions: { getRadius: { duration: 300, enter: () => [0] } }, + }), + [visiblePois, isDark] + ); + + const poiBackgroundLayer = useMemo( + () => + new ScatterplotLayer({ + id: 'poi-background', + data: visiblePois, + getPosition: (d) => [d.lng, d.lat], + getRadius: 14, + radiusUnits: 'pixels', + getFillColor: isDark ? [41, 37, 36, 255] : [255, 255, 255, 255], + getLineColor: (d) => { + const c = POI_GROUP_COLORS[d.group] || POI_DEFAULT_COLOR; + return [c[0], c[1], c[2], 255] as [number, number, number, number]; + }, + getLineWidth: 2.5, + lineWidthUnits: 'pixels', + stroked: true, + pickable: true, + onHover: stablePoiHover, + transitions: { getRadius: { duration: 300, enter: () => [0] } }, + }), + [visiblePois, isDark, stablePoiHover] + ); + + const poiIconLayer = useMemo( () => new IconLayer({ id: 'poi-icons', - data: pois, + data: visiblePois, getPosition: (d) => [d.lng, d.lat], getIcon: (d) => ({ url: emojiToTwemojiUrl(d.emoji), width: 72, height: 72, }), - getSize: 24, - sizeMinPixels: 20, - sizeMaxPixels: 40, - pickable: true, - onHover: stablePoiHover, + getSize: 18, + sizeUnits: 'pixels', + pickable: false, + transitions: { getSize: { duration: 300, enter: () => [0] } }, }), - [pois, stablePoiHover] + [visiblePois] + ); + + // --- Cluster layers --- + const clusterCircleLayer = useMemo( + () => + new ScatterplotLayer({ + id: 'poi-clusters', + data: clusters, + getPosition: (d) => [d.lng, d.lat], + getRadius: (d) => Math.min(30, 14 + Math.sqrt(d.count) * 2), + radiusUnits: 'pixels', + getFillColor: isDark ? [5, 129, 114, 220] : [20, 184, 166, 220], + getLineColor: [255, 255, 255, isDark ? 60 : 120], + getLineWidth: 2, + lineWidthUnits: 'pixels', + stroked: true, + pickable: true, + onHover: stableClusterHover, + transitions: { getRadius: { duration: 300, enter: () => [0] } }, + }), + [clusters, isDark, stableClusterHover] + ); + + const clusterTextLayer = useMemo( + () => + new TextLayer({ + id: 'poi-cluster-text', + data: clusters, + getPosition: (d) => [d.lng, d.lat], + getText: (d) => + d.count >= 1000 ? `${(d.count / 1000).toFixed(1)}k` : String(d.count), + getSize: 12, + getColor: [255, 255, 255, 255], + fontWeight: 700, + fontFamily: 'Inter, system-ui, sans-serif', + getTextAnchor: 'middle', + getAlignmentBaseline: 'center', + sizeUnits: 'pixels', + pickable: false, + }), + [clusters] ); // Marching ants highlight layer for selected hexagon or postcode @@ -511,13 +681,18 @@ export function useDeckLayers({ }); }, [selectedPostcodeGeometry, selectedHexagonId, marchTime]); + const poiLayers = useMemo( + () => [poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer], + [poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer] + ); + const layers = useMemo(() => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const baseLayers: any[] = usePostcodeView ? zoom >= 16 - ? [postcodeLayer, postcodeLabelsLayer, poiLayer] - : [postcodeLayer, poiLayer] - : [hexLayer, poiLayer]; + ? [postcodeLayer, postcodeLabelsLayer, ...poiLayers] + : [postcodeLayer, ...poiLayers] + : [hexLayer, ...poiLayers]; if (marchingAntsLayer) baseLayers.push(marchingAntsLayer); return baseLayers; }, [ @@ -526,7 +701,7 @@ export function useDeckLayers({ hexLayer, postcodeLayer, postcodeLabelsLayer, - poiLayer, + poiLayers, marchingAntsLayer, ]); diff --git a/frontend/src/hooks/useHexagonSelection.ts b/frontend/src/hooks/useHexagonSelection.ts index 2150f55..ee52752 100644 --- a/frontend/src/hooks/useHexagonSelection.ts +++ b/frontend/src/hooks/useHexagonSelection.ts @@ -16,13 +16,20 @@ interface SelectedHexagon { resolution: number; } +interface JourneyDest { + mode: string; + slug: string; +} + interface UseHexagonSelectionOptions { filters: FeatureFilters; features: FeatureMeta[]; resolution: number; + /** First transit destination — used to pick the best central_postcode for journey display. */ + journeyDest?: JourneyDest | null; } -export function useHexagonSelection({ filters, features, resolution }: UseHexagonSelectionOptions) { +export function useHexagonSelection({ filters, features, resolution, journeyDest }: UseHexagonSelectionOptions) { const [selectedHexagon, setSelectedHexagon] = useState(null); const [properties, setProperties] = useState([]); const [propertiesTotal, setPropertiesTotal] = useState(0); @@ -46,11 +53,15 @@ export function useHexagonSelection({ filters, features, resolution }: UseHexago if (fields) { params.set('fields', fields.join(',')); } + if (journeyDest) { + params.set('journey_mode', journeyDest.mode); + params.set('journey_slug', journeyDest.slug); + } const response = await fetch(apiUrl('hexagon-stats', params), authHeaders({ signal })); assertOk(response, 'hexagon-stats'); return (await response.json()) as HexagonStatsResponse; }, - [filters, features] + [filters, features, journeyDest] ); const fetchPostcodeStats = useCallback( diff --git a/frontend/src/hooks/useTravelDestinations.ts b/frontend/src/hooks/useTravelDestinations.ts new file mode 100644 index 0000000..7068222 --- /dev/null +++ b/frontend/src/hooks/useTravelDestinations.ts @@ -0,0 +1,46 @@ +import { useState, useEffect, useRef } from 'react'; +import { authHeaders, logNonAbortError } from '../lib/api'; +import type { TransportMode } from './useTravelTime'; + +export interface Destination { + name: string; + slug: string; + place_type: string; + city?: string; +} + +/** Fetches all travel-time destinations for a mode once, with client-side caching. */ +export function useTravelDestinations(mode: TransportMode) { + const [destinations, setDestinations] = useState([]); + const [loading, setLoading] = useState(false); + const cacheRef = useRef>>({}); + + useEffect(() => { + if (cacheRef.current[mode]) { + setDestinations(cacheRef.current[mode]!); + return; + } + + const controller = new AbortController(); + setLoading(true); + + fetch( + `/api/travel-destinations?mode=${mode}`, + authHeaders({ signal: controller.signal }), + ) + .then((res) => { + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.json(); + }) + .then((data: { destinations: Destination[] }) => { + cacheRef.current[mode] = data.destinations; + setDestinations(data.destinations); + }) + .catch((err) => logNonAbortError('travel destinations', err)) + .finally(() => setLoading(false)); + + return () => controller.abort(); + }, [mode]); + + return { destinations, loading }; +} diff --git a/frontend/src/hooks/useTravelTime.ts b/frontend/src/hooks/useTravelTime.ts index 8f8cc02..2c74f58 100644 --- a/frontend/src/hooks/useTravelTime.ts +++ b/frontend/src/hooks/useTravelTime.ts @@ -47,7 +47,7 @@ export function useTravelTime(initial?: TravelTimeInitial) { (index: number, slug: string, label: string) => { setEntries((prev) => prev.map((entry, i) => - i === index ? { ...entry, slug, label, timeRange: null } : entry + i === index ? { ...entry, slug, label, timeRange: slug ? [0, 120] : null } : entry ) ); }, diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index 2d3e057..3862fcd 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -28,8 +28,8 @@ export const INITIAL_VIEW_STATE: ViewState = { * Returns the H3 resolution to use for a given zoom level. */ export const ZOOM_TO_RESOLUTION_THRESHOLDS = [ - { maxZoom: 7.5, resolution: 5 }, - { maxZoom: 9.5, resolution: 6 }, + { maxZoom: 7, resolution: 5 }, + { maxZoom: 9, resolution: 6 }, { maxZoom: 10.5, resolution: 7 }, { maxZoom: 11.5, resolution: 8 }, { maxZoom: 13, resolution: 9 }, @@ -68,6 +68,41 @@ export const GLYPHS_URL = '/assets/fonts/{fontstack}/{range}.pbf'; /** Twemoji base URL (served locally from public/assets/) */ export const TWEMOJI_BASE = '/assets/twemoji/'; +/** POI group → RGB color for category-coded map markers */ +export const POI_GROUP_COLORS: Record = { + 'Public Transport': [59, 130, 246], + Leisure: [249, 115, 22], + Education: [139, 92, 246], + Health: [239, 68, 68], + 'Emergency Services': [220, 38, 38], + Other: [107, 114, 128], + Groceries: [34, 197, 94], + 'Local Businesses': [245, 158, 11], + Culture: [236, 72, 153], + Services: [6, 182, 212], + Shops: [99, 102, 241], +}; + +/** Default color for unknown POI groups */ +export const POI_DEFAULT_COLOR: [number, number, number] = [107, 114, 128]; + +/** Categories only shown when zoomed in past MINOR_POI_ZOOM_THRESHOLD */ +export const MINOR_POI_CATEGORIES = new Set([ + 'Bus stop', + 'Taxi rank', + 'EV Charging', + 'Playground', +]); + +/** Zoom level below which minor POI categories are hidden */ +export const MINOR_POI_ZOOM_THRESHOLD = 14; + +/** Supercluster grouping radius in pixels */ +export const POI_CLUSTER_RADIUS = 50; + +/** Zoom level at which supercluster stops clustering */ +export const POI_CLUSTER_MAX_ZOOM = 15; + /** * Groups whose features should be collapsed into stacked bar charts. * Keyed by feature group name. Each entry defines one stacked chart. @@ -153,8 +188,8 @@ export const STACKED_ENUM_GROUPS: Record< }, { label: 'Leasehold/Freehold', - feature: 'Leasehold/Freehold', - components: ['Leasehold/Freehold'], + feature: 'Leashold/Freehold', + components: ['Leashold/Freehold'], valueOrder: ['Freehold', 'Leasehold'], valueColors: ['#3b82f6', '#f59e0b'], }, diff --git a/pipeline/transform/poi_proximity.py b/pipeline/transform/poi_proximity.py index b3c6bde..7094b81 100644 --- a/pipeline/transform/poi_proximity.py +++ b/pipeline/transform/poi_proximity.py @@ -1,25 +1,29 @@ -"""Compute POI proximity counts per postcode from ArcGIS + filtered POIs.""" +"""Compute POI proximity counts and distances per postcode from ArcGIS + filtered POIs.""" import argparse from pathlib import Path import polars as pl -from pipeline.utils.poi_counts import count_pois_per_postcode +from pipeline.utils.poi_counts import count_pois_per_postcode, min_distance_per_postcode -# POI category groups for proximity counting. +# POI category groups for proximity counting (2km radius). # Names must match the friendly names produced by transform_poi.py / naptan.py. -POI_GROUPS = { +POI_GROUPS_2KM = { "restaurants": ["Restaurant", "Fast Food"], "groceries": ["Greengrocer", "Supermarket", "Convenience Store"], "parks": ["Park"], - "public_transport": [ - "Metro or Tram stop", - "Rail station", - "Bus stop", - "Bus station", - ], # comes from naptan.py +} + +# Train/tube stations counted at 1km radius +TRAIN_TUBE_GROUP = { + "train_tube": ["Metro or Tram stop", "Rail station"], +} + +# Groups for which to compute distance to nearest POI +DISTANCE_GROUPS = { + "train_tube": ["Metro or Tram stop", "Rail station"], } @@ -46,7 +50,21 @@ def main(): pois = pl.read_parquet(args.pois) - result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=2) + # Count amenity POIs within 2km + counts_2km = count_pois_per_postcode( + postcodes, pois, groups=POI_GROUPS_2KM, radius_km=2 + ) + + # Count train/tube stations within 1km + counts_1km = count_pois_per_postcode( + postcodes, pois, groups=TRAIN_TUBE_GROUP, radius_km=1 + ) + + # Distance to nearest train/tube station + distances = min_distance_per_postcode(postcodes, pois, groups=DISTANCE_GROUPS) + + # Join all results on postcode + result = counts_2km.join(counts_1km, on="postcode").join(distances, on="postcode") result.write_parquet(args.output) size_mb = args.output.stat().st_size / (1024 * 1024) diff --git a/pipeline/transform/postcode_boundaries/voronoi.py b/pipeline/transform/postcode_boundaries/voronoi.py index bc2d494..bec0f86 100644 --- a/pipeline/transform/postcode_boundaries/voronoi.py +++ b/pipeline/transform/postcode_boundaries/voronoi.py @@ -16,8 +16,8 @@ def compute_voronoi_regions( if len(points) == 1: return {postcodes[0]: boundary} - # UPRN coordinates are int64 (BNG grid refs in whole meters). - # Convert to float64 so sub-meter jitter isn't truncated. + # UPRN coordinates are int64 (BNG grid refs in whole metres). + # Convert to float64 so sub-metre jitter isn't truncated. points = points.astype(np.float64) # Deduplicate points, keeping one per (location, postcode) pair. diff --git a/server-rs/src/consts.rs b/server-rs/src/consts.rs index a2fb46c..8981bcf 100644 --- a/server-rs/src/consts.rs +++ b/server-rs/src/consts.rs @@ -7,7 +7,7 @@ pub const H3_REQUEST_MAX: u8 = 12; pub const SERVER_ADDRESS: &str = "0.0.0.0:8001"; pub const GRID_CELL_SIZE: f32 = 0.01; -pub const MAX_POIS_PER_REQUEST: usize = 2500; +pub const MAX_POIS_PER_REQUEST: usize = 10000; pub const MAX_CELLS_PER_REQUEST: usize = 5000; pub const DEFAULT_PROPERTIES_LIMIT: usize = 100; pub const MAX_PROPERTIES_LIMIT: usize = 500; @@ -24,8 +24,9 @@ pub const SERVICE_CALL_TIMEOUT: u64 = 120; /// Users without a license can only query data within these bounds. pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.42, -0.34, 51.60, 0.14); -/// Homepage demo center (lat, lng). Unlicensed hexagon requests are allowed -/// when the center of the requested bounds is within DEMO_CENTER_TOLERANCE of this point. -/// Must match DEMO_VIEW_START in ScrollStory.tsx. +/// Homepage demo center (lat, lng) and tolerance for the license bypass. +/// Hexagon requests centered within this tolerance skip the license check, +/// so the ScrollStory animation works for anonymous visitors. +/// ~0.05° ≈ 5.5 km — covers central London only. pub const DEMO_CENTER: (f64, f64) = (51.51, -0.12); -pub const DEMO_CENTER_TOLERANCE: f64 = 1.0; +pub const DEMO_CENTER_TOLERANCE: f64 = 0.05; diff --git a/server-rs/src/data/travel_time.rs b/server-rs/src/data/travel_time.rs index b0d43a1..ea5f14a 100644 --- a/server-rs/src/data/travel_time.rs +++ b/server-rs/src/data/travel_time.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::sync::Arc; -use anyhow::{bail, Context}; +use anyhow::Context; use parking_lot::Mutex; use polars::lazy::frame::LazyFrame; use rustc_hash::{FxHashMap, FxHashSet}; @@ -167,19 +167,16 @@ impl TravelTimeStore { } } - // Resolve slug to actual filename (may have numeric prefix) + // Resolve slug to actual filename (may have numeric prefix). + // Reject unknown slugs rather than falling back to raw input to prevent path traversal. let file_stem = self .slug_to_file .get(&key) - .map(|val| val.as_str()) - .unwrap_or(slug); + .ok_or_else(|| anyhow::anyhow!("Unknown travel destination: {mode}/{slug}"))?; let path = self .base_dir .join(mode) .join(format!("{}.parquet", file_stem)); - if !path.exists() { - bail!("Travel time file not found: {}", path.display()); - } let df = LazyFrame::scan_parquet(&path, Default::default()) .with_context(|| format!("Failed to scan: {}", path.display()))? diff --git a/server-rs/src/routes/hexagons.rs b/server-rs/src/routes/hexagons.rs index d8e33e9..34ccb9c 100644 --- a/server-rs/src/routes/hexagons.rs +++ b/server-rs/src/routes/hexagons.rs @@ -139,8 +139,6 @@ pub async fn get_hexagons( let (south, west, north, east) = require_bounds(params.bounds).map_err(IntoResponse::into_response)?; - // Allow the homepage demo: check if the center of the requested bounds - // is near the demo view center (51.51, -0.12). let center_lat = (south + north) / 2.0; let center_lng = (west + east) / 2.0; let is_demo_view = (center_lat - DEMO_CENTER.0).abs() <= DEMO_CENTER_TOLERANCE diff --git a/server-rs/src/routes/postcodes.rs b/server-rs/src/routes/postcodes.rs index bde73fd..43f3abc 100644 --- a/server-rs/src/routes/postcodes.rs +++ b/server-rs/src/routes/postcodes.rs @@ -12,10 +12,12 @@ use tracing::info; use crate::aggregation::Aggregator; use crate::auth::OptionalUser; use crate::consts::MAX_CELLS_PER_REQUEST; +use crate::data::travel_time::TravelData; use crate::licensing::check_license_bounds; use crate::parsing::{ bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters, }; +use crate::routes::travel_time::{parse_travel_entries, TravelTimeAgg}; use crate::state::AppState; #[derive(Serialize)] @@ -31,6 +33,8 @@ pub struct PostcodeParams { filters: Option, /// Comma-separated feature names to include in min/max aggregation. fields: Option, + /// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max` + travel: Option, } /// Build a GeoJSON geometry object from postcode polygon rings. @@ -84,10 +88,41 @@ pub async fn get_postcodes( let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index) .map_err(|err| (err.0, err.1).into_response())?; + // Parse travel entries + let travel_entries = params + .travel + .as_deref() + .filter(|val| !val.is_empty()) + .map(parse_travel_entries) + .transpose() + .map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())? + .unwrap_or_default(); + let response = tokio::task::spawn_blocking(move || -> Result { let postcode_data = &state.postcode_data; let t0 = std::time::Instant::now(); + // Load travel time data from precomputed parquet files + let travel_data: Vec = if !travel_entries.is_empty() { + let store = &state.travel_time_store; + travel_entries + .iter() + .map(|entry| { + store + .get(&entry.mode, &entry.slug) + .map_err(|err| format!("Failed to load travel data: {}", err)) + }) + .collect::, _>>()? + } else { + Vec::new() + }; + + let has_travel = !travel_entries.is_empty(); + let travel_field_keys: Vec = travel_entries + .iter() + .map(|te| format!("tt_{}_{}", te.mode, te.slug)) + .collect(); + let num_features = state.data.num_features; let feature_data = &state.data.feature_data; let min_keys = &state.min_keys; @@ -122,9 +157,35 @@ pub async fn get_postcodes( } }); + // Filter postcodes by travel time range (if specified) + if has_travel { + postcode_rows.retain(|&pc_idx, _rows| { + let postcode = &postcode_data.postcodes[pc_idx]; + for (ti, entry) in travel_entries.iter().enumerate() { + if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) { + let minutes = travel_data[ti].get(postcode.as_str()).map(|r| { + if entry.use_best { + r.best_minutes.unwrap_or(r.minutes) + } else { + r.minutes + } + }); + match minutes { + Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {} + _ => return false, + } + } + } + true + }); + } + // Aggregate for each postcode that has properties in bounds // (polygon intersection check happens later when building response) let mut postcode_aggs: FxHashMap = FxHashMap::default(); + // Travel time aggregation per postcode + let mut travel_aggs: FxHashMap> = FxHashMap::default(); + for (&pc_idx, rows) in &postcode_rows { let agg = postcode_aggs .entry(pc_idx) @@ -136,6 +197,24 @@ pub async fn get_postcodes( agg.add_row(feature_data, row, num_features); } } + + // Aggregate travel times for this postcode + if has_travel { + let postcode = &postcode_data.postcodes[pc_idx]; + let tt_aggs = travel_aggs + .entry(pc_idx) + .or_insert_with(|| (0..travel_entries.len()).map(|_| TravelTimeAgg::new()).collect()); + for (ti, entry) in travel_entries.iter().enumerate() { + if let Some(row_data) = travel_data[ti].get(postcode.as_str()) { + let minutes = if entry.use_best { + row_data.best_minutes.unwrap_or(row_data.minutes) + } else { + row_data.minutes + }; + tt_aggs[ti].add(minutes as f32); + } + } + } } // Build response, filtering postcodes to only those whose polygon intersects query bounds @@ -218,6 +297,25 @@ pub async fn get_postcodes( } } + // Add travel time aggregation fields + if let Some(tt_aggs) = travel_aggs.get(&pc_idx) { + for (ti, agg) in tt_aggs.iter().enumerate() { + if agg.count > 0 { + let key = &travel_field_keys[ti]; + let avg = agg.sum / agg.count as f64; + if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) { + props.insert(format!("min_{key}"), Value::Number(nm)); + } + if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) { + props.insert(format!("max_{key}"), Value::Number(nm)); + } + if let Some(nm) = serde_json::Number::from_f64(avg) { + props.insert(format!("avg_{key}"), Value::Number(nm)); + } + } + } + } + // Build GeoJSON Feature let mut feature = Map::new(); feature.insert("type".into(), Value::String("Feature".into())); @@ -241,6 +339,7 @@ pub async fn get_postcodes( bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east), filters = num_filters, filters_raw = filters_str.as_deref().unwrap_or("-"), + travel_entries = travel_entries.len(), total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0), "GET /api/postcodes" ); diff --git a/server-rs/src/routes/properties.rs b/server-rs/src/routes/properties.rs index df5a4dd..22654a9 100644 --- a/server-rs/src/routes/properties.rs +++ b/server-rs/src/routes/properties.rs @@ -130,7 +130,7 @@ pub fn build_property( feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form", ), duration: lookup_enum_value( - feature_name_to_index, feature_data, num_features, enum_values, row, "Leashold/Freehold", + feature_name_to_index, feature_data, num_features, enum_values, row, "Leasehold/Freehold", ), current_energy_rating: lookup_enum_value( feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating", diff --git a/server-rs/src/routes/shorten.rs b/server-rs/src/routes/shorten.rs index ad8e4d1..d4feaec 100644 --- a/server-rs/src/routes/shorten.rs +++ b/server-rs/src/routes/shorten.rs @@ -95,6 +95,13 @@ pub async fn post_shorten(state: Arc, Json(req): Json) } pub async fn get_short_url(state: Arc, Path(code): Path) -> Response { + if code.is_empty() + || code.len() > 20 + || !code.bytes().all(|b| b.is_ascii_alphanumeric()) + { + return StatusCode::BAD_REQUEST.into_response(); + } + let pb_url = state.pocketbase_url.trim_end_matches('/'); let token = match auth_superuser( diff --git a/server-rs/src/routes/travel_time.rs b/server-rs/src/routes/travel_time.rs index a98edaf..9437a0e 100644 --- a/server-rs/src/routes/travel_time.rs +++ b/server-rs/src/routes/travel_time.rs @@ -1,4 +1,62 @@ -/// Per-hex-cell travel time aggregation. +/// A parsed travel time entry from the `travel` query parameter. +pub struct TravelEntry { + pub mode: String, + pub slug: String, + pub use_best: bool, + pub filter_min: Option, + pub filter_max: Option, +} + +/// Parse `travel` param into a list of travel entries. +/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max` +pub fn parse_travel_entries(travel_str: &str) -> Result, String> { + let mut entries = Vec::new(); + let mut seen_keys = Vec::new(); + for segment in travel_str.split('|') { + let parts: Vec<&str> = segment.split(':').collect(); + if parts.len() < 2 { + return Err(format!( + "each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'", + segment + )); + } + let mode = parts[0].trim().to_string(); + let slug = parts[1].trim().to_string(); + + let use_best = parts.len() >= 3 && parts[2].trim() == "best"; + let filter_offset = if use_best { 1 } else { 0 }; + + let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset { + let min: f32 = parts[2 + filter_offset] + .trim() + .parse() + .map_err(|_| format!("invalid travel filter min in '{}'", segment))?; + let max: f32 = parts[3 + filter_offset] + .trim() + .parse() + .map_err(|_| format!("invalid travel filter max in '{}'", segment))?; + (Some(min), Some(max)) + } else { + (None, None) + }; + + let key = format!("{}:{}", mode, slug); + if seen_keys.contains(&key) { + return Err(format!("duplicate travel entry '{}'", key)); + } + seen_keys.push(key); + entries.push(TravelEntry { + mode, + slug, + use_best, + filter_min, + filter_max, + }); + } + Ok(entries) +} + +/// Per-cell travel time aggregation. pub struct TravelTimeAgg { pub min: f32, pub max: f32,