More fixes

This commit is contained in:
Andras Schmelczer 2026-03-12 20:27:04 +00:00
parent 791bc6976b
commit 14a3555cf1
21 changed files with 549 additions and 99 deletions

1
.gitignore vendored
View file

@ -1,6 +1,5 @@
.venv
.claude
tfl_journey_client
**/node_modules
**/__pycache__
**/dist

View file

@ -51,6 +51,7 @@ PLACES := $(DATA_DIR)/places.parquet
LISTINGS_BUY := $(DATA_DIR)/online_listings_buy.parquet
LISTINGS_RENT := $(DATA_DIR)/online_listings_rent.parquet
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
RM_OUTCODES := frontend/src/lib/rightmove-outcodes.json
# Sentinel files for directory targets (Make doesn't track directories well)
GEOSURE_STAMP := $(GEOSURE_DIR)/.done
@ -64,7 +65,7 @@ PMTILES_VERSION := 1.22.3
download-arcgis download-price-paid download-deprivation download-ethnicity \
download-naptan download-pois download-ofsted download-broadband download-rental-prices \
download-postcodes download-geosure download-noise download-inspire \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population download-rightmove-outcodes \
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
transform-school-proximity transform-geosure transform-postcode-boundaries \
generate-postcode-boundaries
@ -92,6 +93,7 @@ download-greenspace: $(GREENSPACE)
download-pbf: $(PBF)
download-places: $(PLACES)
download-lsoa-population: $(LSOA_POP)
download-rightmove-outcodes: $(RM_OUTCODES)
transform-pois: $(POIS_FILTERED)
transform-epc-pp: $(EPC_PP)
transform-crime: $(CRIME)
@ -187,6 +189,9 @@ $(PLACES): $(PBF)
$(LSOA_POP):
uv run python -m pipeline.download.lsoa_population --output $@
$(RM_OUTCODES): $(MERGE_STAMP)
uv run python -m pipeline.download.rightmove_outcodes --postcodes $(POSTCODES_PQ) --output $@
# ── Transforms ────────────────────────────────────────────────────────────────
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN)

View file

@ -14,7 +14,10 @@ SEED = 42
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
# Whether to run a scrape immediately on startup
RUN_ON_STARTUP = False
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
# Enable/disable individual sources
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes")
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes")
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"

View file

@ -56,7 +56,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
"Address per Property Register": [
p["Address per Property Register"] for p in properties
],
"Leashold/Freehold": [p["Leashold/Freehold"] for p in properties],
"Leasehold/Freehold": [p["Leasehold/Freehold"] for p in properties],
"Property type": [p["Property type"] for p in properties],
"Property sub-type": [p["Property sub-type"] for p in properties],
"Price qualifier": [p["Price qualifier"] for p in properties],
@ -78,7 +78,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
"lat": pl.Float64,
"Postcode": pl.Utf8,
"Address per Property Register": pl.Utf8,
"Leashold/Freehold": pl.Utf8,
"Leasehold/Freehold": pl.Utf8,
"Property type": pl.Utf8,
"Property sub-type": pl.Utf8,
"Price qualifier": pl.Utf8,

View file

@ -59,16 +59,6 @@ export default memo(function HoverCard({ x, y, id, isPostcode, data, filters, fe
transform: 'translate(-50%, -100%)',
}}
>
{/* Arrow */}
<div
className="absolute w-3 h-3 bg-white dark:bg-warm-800 rotate-45"
style={{
left: '50%',
bottom: -6,
transform: 'translateX(-50%)',
}}
/>
<div className="relative">
{/* Header */}
<div className="flex items-center justify-between gap-2 mb-1">

View file

@ -14,7 +14,7 @@ import type {
} from '../../types';
import { zoomToResolution, getBoundsFromViewState, getMapStyle } from '../../lib/map-utils';
import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS } from '../../lib/consts';
import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS, POI_GROUP_COLORS, POI_DEFAULT_COLOR } from '../../lib/consts';
import LocationSearch, { type SearchedLocation } from './LocationSearch';
import MapLegend from './MapLegend';
import HoverCard from './HoverCard';
@ -263,25 +263,49 @@ export default memo(function Map({
))}
{popupInfo && (
<div
className="absolute bg-white dark:bg-warm-800 rounded shadow-lg p-2 text-sm dark:text-white"
className="absolute bg-white dark:bg-warm-800 rounded-lg shadow-lg text-sm dark:text-white pointer-events-none"
style={{
left: popupInfo.x,
top: popupInfo.y - 40,
top: popupInfo.y - 50,
transform: 'translateX(-50%)',
zIndex: 9999,
}}
>
<strong className="dark:text-white">{popupInfo.name}</strong>
<div className="text-warm-500 dark:text-warm-300 text-xs">{popupInfo.category}</div>
{osmIdToUrl(popupInfo.id) && (
<a
href={osmIdToUrl(popupInfo.id)!}
target="_blank"
rel="noopener noreferrer"
className="text-teal-600 dark:text-teal-400 hover:text-teal-800 dark:hover:text-teal-300 text-xs"
>
View on OSM
</a>
{popupInfo.isCluster ? (
<div className="px-3 py-2 text-center">
<div className="text-lg font-bold text-teal-600 dark:text-teal-400">
{popupInfo.clusterCount}
</div>
<div className="text-warm-500 dark:text-warm-400 text-xs">places</div>
</div>
) : (
<div className="px-3 py-2">
<div className="flex items-center gap-2">
<span className="text-lg leading-none">{popupInfo.emoji}</span>
<div>
<div className="font-semibold dark:text-warm-100">{popupInfo.name}</div>
<div className="flex items-center gap-1.5 text-xs text-warm-500 dark:text-warm-400">
<span
className="inline-block w-2 h-2 rounded-full flex-shrink-0"
style={{
backgroundColor: `rgb(${(POI_GROUP_COLORS[popupInfo.group] || POI_DEFAULT_COLOR).join(',')})`,
}}
/>
{popupInfo.category}
</div>
</div>
</div>
{osmIdToUrl(popupInfo.id) && (
<a
href={osmIdToUrl(popupInfo.id)!}
target="_blank"
rel="noopener noreferrer"
className="text-teal-600 dark:text-teal-400 hover:text-teal-800 dark:hover:text-teal-300 text-xs mt-1 block pointer-events-auto"
>
View on OSM
</a>
)}
</div>
)}
</div>
)}

View file

@ -135,17 +135,6 @@ function PropertyLoadingSkeleton() {
);
}
const LISTING_STATUS_STYLES: Record<string, string> = {
'For sale': 'bg-teal-100 text-teal-800 dark:bg-teal-900/40 dark:text-teal-300',
'For rent': 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300',
'Historical sale': 'bg-warm-100 text-warm-600 dark:bg-warm-700 dark:text-warm-300',
};
function ListingStatusBadge({ status }: { status: string }) {
const style = LISTING_STATUS_STYLES[status] ?? LISTING_STATUS_STYLES['Historical sale'];
return <span className={`text-xs font-medium px-1.5 py-0.5 rounded ${style}`}>{status}</span>;
}
function PropertyCard({ property }: { property: Property }) {
const price = getNum(property, 'Last known price');
const estimatedPrice = getNum(property, 'Estimated current price');
@ -161,18 +150,13 @@ function PropertyCard({ property }: { property: Property }) {
const bathrooms = getNum(property, 'Bathrooms');
const listingDate = getNum(property, 'Listing date');
const listingStatus = property.listing_status;
return (
<div className="p-4 border-b border-warm-100 dark:border-warm-800 hover:bg-warm-50 dark:hover:bg-warm-800">
<div className="flex items-start justify-between gap-2">
<div>
<div className="font-semibold dark:text-warm-100">
{property.address || 'Unknown Address'}
</div>
<div className="text-sm text-warm-600 dark:text-warm-400">{property.postcode}</div>
<div>
<div className="font-semibold dark:text-warm-100">
{property.address || 'Unknown Address'}
</div>
{listingStatus && <ListingStatusBadge status={listingStatus} />}
<div className="text-sm text-warm-600 dark:text-warm-400">{property.postcode}</div>
</div>
{property.property_sub_type && (

View file

@ -1,7 +1,8 @@
import { useCallback, useRef, useState, useMemo, useEffect } from 'react';
import { H3HexagonLayer } from '@deck.gl/geo-layers';
import { GeoJsonLayer, IconLayer, TextLayer } from '@deck.gl/layers';
import { GeoJsonLayer, IconLayer, TextLayer, ScatterplotLayer } from '@deck.gl/layers';
import { cellToBoundary } from 'h3-js';
import Supercluster from 'supercluster';
import type { PickingInfo } from '@deck.gl/core';
import type {
HexagonData,
@ -12,7 +13,16 @@ import type {
FeatureMeta,
Bounds,
} from '../types';
import { DENSITY_GRADIENT, DENSITY_GRADIENT_DARK } from '../lib/consts';
import {
DENSITY_GRADIENT,
DENSITY_GRADIENT_DARK,
POI_GROUP_COLORS,
POI_DEFAULT_COLOR,
MINOR_POI_CATEGORIES,
MINOR_POI_ZOOM_THRESHOLD,
POI_CLUSTER_RADIUS,
POI_CLUSTER_MAX_ZOOM,
} from '../lib/consts';
import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils';
import {
type TravelTimeEntry,
@ -55,7 +65,18 @@ interface PopupInfo {
y: number;
name: string;
category: string;
group: string;
emoji: string;
id: string;
isCluster?: boolean;
clusterCount?: number;
}
interface ClusterPoint {
lng: number;
lat: number;
count: number;
clusterId: number;
}
export function useDeckLayers({
@ -204,6 +225,8 @@ export function useDeckLayers({
y: info.y,
name: info.object.name,
category: info.object.category,
group: info.object.group,
emoji: info.object.emoji,
id: info.object.id,
});
} else {
@ -217,6 +240,30 @@ export function useDeckLayers({
handlePoiHoverRef.current(info);
}, []);
const handleClusterHover = useCallback((info: PickingInfo<ClusterPoint>) => {
if (info.object && info.x !== undefined && info.y !== undefined) {
setPopupInfo({
x: info.x,
y: info.y,
name: `${info.object.count} places`,
category: 'Zoom in to see details',
group: '',
emoji: '',
id: '',
isCluster: true,
clusterCount: info.object.count,
});
} else {
setPopupInfo(null);
}
}, []);
const handleClusterHoverRef = useRef(handleClusterHover);
handleClusterHoverRef.current = handleClusterHover;
const stableClusterHover = useCallback((info: PickingInfo<ClusterPoint>) => {
handleClusterHoverRef.current(info);
}, []);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const handlePostcodeClick = useCallback((info: PickingInfo<any>) => {
const pc = info.object?.properties?.postcode;
@ -461,24 +508,147 @@ export function useDeckLayers({
[postcodeData, theme]
);
const poiLayer = useMemo(
// --- POI clustering ---
const clusterIndex = useMemo(() => {
if (pois.length === 0) return null;
const index = new Supercluster<POI>({
radius: POI_CLUSTER_RADIUS,
maxZoom: POI_CLUSTER_MAX_ZOOM,
});
const features: Supercluster.PointFeature<POI>[] = pois.map((poi) => ({
type: 'Feature',
geometry: { type: 'Point', coordinates: [poi.lng, poi.lat] },
properties: poi,
}));
index.load(features);
return index;
}, [pois]);
const clusterZoom = Math.floor(zoom);
const showMinorPois = zoom >= MINOR_POI_ZOOM_THRESHOLD;
const { visiblePois, clusters } = useMemo(() => {
if (!clusterIndex || pois.length === 0) {
return { visiblePois: [] as POI[], clusters: [] as ClusterPoint[] };
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const allFeatures = clusterIndex.getClusters([-180, -85, 180, 85], clusterZoom) as any[];
const individual: POI[] = [];
const clusterPoints: ClusterPoint[] = [];
for (const feature of allFeatures) {
if (feature.properties.cluster) {
clusterPoints.push({
lng: feature.geometry.coordinates[0],
lat: feature.geometry.coordinates[1],
count: feature.properties.point_count,
clusterId: feature.properties.cluster_id,
});
} else {
const poi = feature.properties as POI;
if (!showMinorPois && MINOR_POI_CATEGORIES.has(poi.category)) continue;
individual.push(poi);
}
}
return { visiblePois: individual, clusters: clusterPoints };
}, [clusterIndex, clusterZoom, showMinorPois, pois]);
// --- Individual POI layers (shadow → background → emoji) ---
const poiShadowLayer = useMemo(
() =>
new ScatterplotLayer<POI>({
id: 'poi-shadow',
data: visiblePois,
getPosition: (d) => [d.lng, d.lat],
getRadius: 16,
radiusUnits: 'pixels',
getFillColor: isDark ? [0, 0, 0, 50] : [0, 0, 0, 25],
pickable: false,
transitions: { getRadius: { duration: 300, enter: () => [0] } },
}),
[visiblePois, isDark]
);
const poiBackgroundLayer = useMemo(
() =>
new ScatterplotLayer<POI>({
id: 'poi-background',
data: visiblePois,
getPosition: (d) => [d.lng, d.lat],
getRadius: 14,
radiusUnits: 'pixels',
getFillColor: isDark ? [41, 37, 36, 255] : [255, 255, 255, 255],
getLineColor: (d) => {
const c = POI_GROUP_COLORS[d.group] || POI_DEFAULT_COLOR;
return [c[0], c[1], c[2], 255] as [number, number, number, number];
},
getLineWidth: 2.5,
lineWidthUnits: 'pixels',
stroked: true,
pickable: true,
onHover: stablePoiHover,
transitions: { getRadius: { duration: 300, enter: () => [0] } },
}),
[visiblePois, isDark, stablePoiHover]
);
const poiIconLayer = useMemo(
() =>
new IconLayer<POI>({
id: 'poi-icons',
data: pois,
data: visiblePois,
getPosition: (d) => [d.lng, d.lat],
getIcon: (d) => ({
url: emojiToTwemojiUrl(d.emoji),
width: 72,
height: 72,
}),
getSize: 24,
sizeMinPixels: 20,
sizeMaxPixels: 40,
pickable: true,
onHover: stablePoiHover,
getSize: 18,
sizeUnits: 'pixels',
pickable: false,
transitions: { getSize: { duration: 300, enter: () => [0] } },
}),
[pois, stablePoiHover]
[visiblePois]
);
// --- Cluster layers ---
const clusterCircleLayer = useMemo(
() =>
new ScatterplotLayer<ClusterPoint>({
id: 'poi-clusters',
data: clusters,
getPosition: (d) => [d.lng, d.lat],
getRadius: (d) => Math.min(30, 14 + Math.sqrt(d.count) * 2),
radiusUnits: 'pixels',
getFillColor: isDark ? [5, 129, 114, 220] : [20, 184, 166, 220],
getLineColor: [255, 255, 255, isDark ? 60 : 120],
getLineWidth: 2,
lineWidthUnits: 'pixels',
stroked: true,
pickable: true,
onHover: stableClusterHover,
transitions: { getRadius: { duration: 300, enter: () => [0] } },
}),
[clusters, isDark, stableClusterHover]
);
const clusterTextLayer = useMemo(
() =>
new TextLayer<ClusterPoint>({
id: 'poi-cluster-text',
data: clusters,
getPosition: (d) => [d.lng, d.lat],
getText: (d) =>
d.count >= 1000 ? `${(d.count / 1000).toFixed(1)}k` : String(d.count),
getSize: 12,
getColor: [255, 255, 255, 255],
fontWeight: 700,
fontFamily: 'Inter, system-ui, sans-serif',
getTextAnchor: 'middle',
getAlignmentBaseline: 'center',
sizeUnits: 'pixels',
pickable: false,
}),
[clusters]
);
// Marching ants highlight layer for selected hexagon or postcode
@ -511,13 +681,18 @@ export function useDeckLayers({
});
}, [selectedPostcodeGeometry, selectedHexagonId, marchTime]);
const poiLayers = useMemo(
() => [poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer],
[poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer]
);
const layers = useMemo(() => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const baseLayers: any[] = usePostcodeView
? zoom >= 16
? [postcodeLayer, postcodeLabelsLayer, poiLayer]
: [postcodeLayer, poiLayer]
: [hexLayer, poiLayer];
? [postcodeLayer, postcodeLabelsLayer, ...poiLayers]
: [postcodeLayer, ...poiLayers]
: [hexLayer, ...poiLayers];
if (marchingAntsLayer) baseLayers.push(marchingAntsLayer);
return baseLayers;
}, [
@ -526,7 +701,7 @@ export function useDeckLayers({
hexLayer,
postcodeLayer,
postcodeLabelsLayer,
poiLayer,
poiLayers,
marchingAntsLayer,
]);

View file

@ -16,13 +16,20 @@ interface SelectedHexagon {
resolution: number;
}
interface JourneyDest {
mode: string;
slug: string;
}
interface UseHexagonSelectionOptions {
filters: FeatureFilters;
features: FeatureMeta[];
resolution: number;
/** First transit destination — used to pick the best central_postcode for journey display. */
journeyDest?: JourneyDest | null;
}
export function useHexagonSelection({ filters, features, resolution }: UseHexagonSelectionOptions) {
export function useHexagonSelection({ filters, features, resolution, journeyDest }: UseHexagonSelectionOptions) {
const [selectedHexagon, setSelectedHexagon] = useState<SelectedHexagon | null>(null);
const [properties, setProperties] = useState<Property[]>([]);
const [propertiesTotal, setPropertiesTotal] = useState(0);
@ -46,11 +53,15 @@ export function useHexagonSelection({ filters, features, resolution }: UseHexago
if (fields) {
params.set('fields', fields.join(','));
}
if (journeyDest) {
params.set('journey_mode', journeyDest.mode);
params.set('journey_slug', journeyDest.slug);
}
const response = await fetch(apiUrl('hexagon-stats', params), authHeaders({ signal }));
assertOk(response, 'hexagon-stats');
return (await response.json()) as HexagonStatsResponse;
},
[filters, features]
[filters, features, journeyDest]
);
const fetchPostcodeStats = useCallback(

View file

@ -0,0 +1,46 @@
import { useState, useEffect, useRef } from 'react';
import { authHeaders, logNonAbortError } from '../lib/api';
import type { TransportMode } from './useTravelTime';
export interface Destination {
name: string;
slug: string;
place_type: string;
city?: string;
}
/** Fetches all travel-time destinations for a mode once, with client-side caching. */
export function useTravelDestinations(mode: TransportMode) {
const [destinations, setDestinations] = useState<Destination[]>([]);
const [loading, setLoading] = useState(false);
const cacheRef = useRef<Partial<Record<TransportMode, Destination[]>>>({});
useEffect(() => {
if (cacheRef.current[mode]) {
setDestinations(cacheRef.current[mode]!);
return;
}
const controller = new AbortController();
setLoading(true);
fetch(
`/api/travel-destinations?mode=${mode}`,
authHeaders({ signal: controller.signal }),
)
.then((res) => {
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
})
.then((data: { destinations: Destination[] }) => {
cacheRef.current[mode] = data.destinations;
setDestinations(data.destinations);
})
.catch((err) => logNonAbortError('travel destinations', err))
.finally(() => setLoading(false));
return () => controller.abort();
}, [mode]);
return { destinations, loading };
}

View file

@ -47,7 +47,7 @@ export function useTravelTime(initial?: TravelTimeInitial) {
(index: number, slug: string, label: string) => {
setEntries((prev) =>
prev.map((entry, i) =>
i === index ? { ...entry, slug, label, timeRange: null } : entry
i === index ? { ...entry, slug, label, timeRange: slug ? [0, 120] : null } : entry
)
);
},

View file

@ -28,8 +28,8 @@ export const INITIAL_VIEW_STATE: ViewState = {
* Returns the H3 resolution to use for a given zoom level.
*/
export const ZOOM_TO_RESOLUTION_THRESHOLDS = [
{ maxZoom: 7.5, resolution: 5 },
{ maxZoom: 9.5, resolution: 6 },
{ maxZoom: 7, resolution: 5 },
{ maxZoom: 9, resolution: 6 },
{ maxZoom: 10.5, resolution: 7 },
{ maxZoom: 11.5, resolution: 8 },
{ maxZoom: 13, resolution: 9 },
@ -68,6 +68,41 @@ export const GLYPHS_URL = '/assets/fonts/{fontstack}/{range}.pbf';
/** Twemoji base URL (served locally from public/assets/) */
export const TWEMOJI_BASE = '/assets/twemoji/';
/** POI group → RGB color for category-coded map markers */
export const POI_GROUP_COLORS: Record<string, [number, number, number]> = {
'Public Transport': [59, 130, 246],
Leisure: [249, 115, 22],
Education: [139, 92, 246],
Health: [239, 68, 68],
'Emergency Services': [220, 38, 38],
Other: [107, 114, 128],
Groceries: [34, 197, 94],
'Local Businesses': [245, 158, 11],
Culture: [236, 72, 153],
Services: [6, 182, 212],
Shops: [99, 102, 241],
};
/** Default color for unknown POI groups */
export const POI_DEFAULT_COLOR: [number, number, number] = [107, 114, 128];
/** Categories only shown when zoomed in past MINOR_POI_ZOOM_THRESHOLD */
export const MINOR_POI_CATEGORIES = new Set([
'Bus stop',
'Taxi rank',
'EV Charging',
'Playground',
]);
/** Zoom level below which minor POI categories are hidden */
export const MINOR_POI_ZOOM_THRESHOLD = 14;
/** Supercluster grouping radius in pixels */
export const POI_CLUSTER_RADIUS = 50;
/** Zoom level at which supercluster stops clustering */
export const POI_CLUSTER_MAX_ZOOM = 15;
/**
* Groups whose features should be collapsed into stacked bar charts.
* Keyed by feature group name. Each entry defines one stacked chart.
@ -153,8 +188,8 @@ export const STACKED_ENUM_GROUPS: Record<
},
{
label: 'Leasehold/Freehold',
feature: 'Leasehold/Freehold',
components: ['Leasehold/Freehold'],
feature: 'Leashold/Freehold',
components: ['Leashold/Freehold'],
valueOrder: ['Freehold', 'Leasehold'],
valueColors: ['#3b82f6', '#f59e0b'],
},

View file

@ -1,25 +1,29 @@
"""Compute POI proximity counts per postcode from ArcGIS + filtered POIs."""
"""Compute POI proximity counts and distances per postcode from ArcGIS + filtered POIs."""
import argparse
from pathlib import Path
import polars as pl
from pipeline.utils.poi_counts import count_pois_per_postcode
from pipeline.utils.poi_counts import count_pois_per_postcode, min_distance_per_postcode
# POI category groups for proximity counting.
# POI category groups for proximity counting (2km radius).
# Names must match the friendly names produced by transform_poi.py / naptan.py.
POI_GROUPS = {
POI_GROUPS_2KM = {
"restaurants": ["Restaurant", "Fast Food"],
"groceries": ["Greengrocer", "Supermarket", "Convenience Store"],
"parks": ["Park"],
"public_transport": [
"Metro or Tram stop",
"Rail station",
"Bus stop",
"Bus station",
], # comes from naptan.py
}
# Train/tube stations counted at 1km radius
TRAIN_TUBE_GROUP = {
"train_tube": ["Metro or Tram stop", "Rail station"],
}
# Groups for which to compute distance to nearest POI
DISTANCE_GROUPS = {
"train_tube": ["Metro or Tram stop", "Rail station"],
}
@ -46,7 +50,21 @@ def main():
pois = pl.read_parquet(args.pois)
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=2)
# Count amenity POIs within 2km
counts_2km = count_pois_per_postcode(
postcodes, pois, groups=POI_GROUPS_2KM, radius_km=2
)
# Count train/tube stations within 1km
counts_1km = count_pois_per_postcode(
postcodes, pois, groups=TRAIN_TUBE_GROUP, radius_km=1
)
# Distance to nearest train/tube station
distances = min_distance_per_postcode(postcodes, pois, groups=DISTANCE_GROUPS)
# Join all results on postcode
result = counts_2km.join(counts_1km, on="postcode").join(distances, on="postcode")
result.write_parquet(args.output)
size_mb = args.output.stat().st_size / (1024 * 1024)

View file

@ -16,8 +16,8 @@ def compute_voronoi_regions(
if len(points) == 1:
return {postcodes[0]: boundary}
# UPRN coordinates are int64 (BNG grid refs in whole meters).
# Convert to float64 so sub-meter jitter isn't truncated.
# UPRN coordinates are int64 (BNG grid refs in whole metres).
# Convert to float64 so sub-metre jitter isn't truncated.
points = points.astype(np.float64)
# Deduplicate points, keeping one per (location, postcode) pair.

View file

@ -7,7 +7,7 @@ pub const H3_REQUEST_MAX: u8 = 12;
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
pub const GRID_CELL_SIZE: f32 = 0.01;
pub const MAX_POIS_PER_REQUEST: usize = 2500;
pub const MAX_POIS_PER_REQUEST: usize = 10000;
pub const MAX_CELLS_PER_REQUEST: usize = 5000;
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
pub const MAX_PROPERTIES_LIMIT: usize = 500;
@ -24,8 +24,9 @@ pub const SERVICE_CALL_TIMEOUT: u64 = 120;
/// Users without a license can only query data within these bounds.
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.42, -0.34, 51.60, 0.14);
/// Homepage demo center (lat, lng). Unlicensed hexagon requests are allowed
/// when the center of the requested bounds is within DEMO_CENTER_TOLERANCE of this point.
/// Must match DEMO_VIEW_START in ScrollStory.tsx.
/// Homepage demo center (lat, lng) and tolerance for the license bypass.
/// Hexagon requests centered within this tolerance skip the license check,
/// so the ScrollStory animation works for anonymous visitors.
/// ~0.05° ≈ 5.5 km — covers central London only.
pub const DEMO_CENTER: (f64, f64) = (51.51, -0.12);
pub const DEMO_CENTER_TOLERANCE: f64 = 1.0;
pub const DEMO_CENTER_TOLERANCE: f64 = 0.05;

View file

@ -2,7 +2,7 @@ use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{bail, Context};
use anyhow::Context;
use parking_lot::Mutex;
use polars::lazy::frame::LazyFrame;
use rustc_hash::{FxHashMap, FxHashSet};
@ -167,19 +167,16 @@ impl TravelTimeStore {
}
}
// Resolve slug to actual filename (may have numeric prefix)
// Resolve slug to actual filename (may have numeric prefix).
// Reject unknown slugs rather than falling back to raw input to prevent path traversal.
let file_stem = self
.slug_to_file
.get(&key)
.map(|val| val.as_str())
.unwrap_or(slug);
.ok_or_else(|| anyhow::anyhow!("Unknown travel destination: {mode}/{slug}"))?;
let path = self
.base_dir
.join(mode)
.join(format!("{}.parquet", file_stem));
if !path.exists() {
bail!("Travel time file not found: {}", path.display());
}
let df = LazyFrame::scan_parquet(&path, Default::default())
.with_context(|| format!("Failed to scan: {}", path.display()))?

View file

@ -139,8 +139,6 @@ pub async fn get_hexagons(
let (south, west, north, east) =
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
// Allow the homepage demo: check if the center of the requested bounds
// is near the demo view center (51.51, -0.12).
let center_lat = (south + north) / 2.0;
let center_lng = (west + east) / 2.0;
let is_demo_view = (center_lat - DEMO_CENTER.0).abs() <= DEMO_CENTER_TOLERANCE

View file

@ -12,10 +12,12 @@ use tracing::info;
use crate::aggregation::Aggregator;
use crate::auth::OptionalUser;
use crate::consts::MAX_CELLS_PER_REQUEST;
use crate::data::travel_time::TravelData;
use crate::licensing::check_license_bounds;
use crate::parsing::{
bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters,
};
use crate::routes::travel_time::{parse_travel_entries, TravelTimeAgg};
use crate::state::AppState;
#[derive(Serialize)]
@ -31,6 +33,8 @@ pub struct PostcodeParams {
filters: Option<String>,
/// Comma-separated feature names to include in min/max aggregation.
fields: Option<String>,
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
travel: Option<String>,
}
/// Build a GeoJSON geometry object from postcode polygon rings.
@ -84,10 +88,41 @@ pub async fn get_postcodes(
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index)
.map_err(|err| (err.0, err.1).into_response())?;
// Parse travel entries
let travel_entries = params
.travel
.as_deref()
.filter(|val| !val.is_empty())
.map(parse_travel_entries)
.transpose()
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
.unwrap_or_default();
let response = tokio::task::spawn_blocking(move || -> Result<PostcodesResponse, String> {
let postcode_data = &state.postcode_data;
let t0 = std::time::Instant::now();
// Load travel time data from precomputed parquet files
let travel_data: Vec<TravelData> = if !travel_entries.is_empty() {
let store = &state.travel_time_store;
travel_entries
.iter()
.map(|entry| {
store
.get(&entry.mode, &entry.slug)
.map_err(|err| format!("Failed to load travel data: {}", err))
})
.collect::<Result<Vec<_>, _>>()?
} else {
Vec::new()
};
let has_travel = !travel_entries.is_empty();
let travel_field_keys: Vec<String> = travel_entries
.iter()
.map(|te| format!("tt_{}_{}", te.mode, te.slug))
.collect();
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let min_keys = &state.min_keys;
@ -122,9 +157,35 @@ pub async fn get_postcodes(
}
});
// Filter postcodes by travel time range (if specified)
if has_travel {
postcode_rows.retain(|&pc_idx, _rows| {
let postcode = &postcode_data.postcodes[pc_idx];
for (ti, entry) in travel_entries.iter().enumerate() {
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
let minutes = travel_data[ti].get(postcode.as_str()).map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
match minutes {
Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => return false,
}
}
}
true
});
}
// Aggregate for each postcode that has properties in bounds
// (polygon intersection check happens later when building response)
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
// Travel time aggregation per postcode
let mut travel_aggs: FxHashMap<usize, Vec<TravelTimeAgg>> = FxHashMap::default();
for (&pc_idx, rows) in &postcode_rows {
let agg = postcode_aggs
.entry(pc_idx)
@ -136,6 +197,24 @@ pub async fn get_postcodes(
agg.add_row(feature_data, row, num_features);
}
}
// Aggregate travel times for this postcode
if has_travel {
let postcode = &postcode_data.postcodes[pc_idx];
let tt_aggs = travel_aggs
.entry(pc_idx)
.or_insert_with(|| (0..travel_entries.len()).map(|_| TravelTimeAgg::new()).collect());
for (ti, entry) in travel_entries.iter().enumerate() {
if let Some(row_data) = travel_data[ti].get(postcode.as_str()) {
let minutes = if entry.use_best {
row_data.best_minutes.unwrap_or(row_data.minutes)
} else {
row_data.minutes
};
tt_aggs[ti].add(minutes as f32);
}
}
}
}
// Build response, filtering postcodes to only those whose polygon intersects query bounds
@ -218,6 +297,25 @@ pub async fn get_postcodes(
}
}
// Add travel time aggregation fields
if let Some(tt_aggs) = travel_aggs.get(&pc_idx) {
for (ti, agg) in tt_aggs.iter().enumerate() {
if agg.count > 0 {
let key = &travel_field_keys[ti];
let avg = agg.sum / agg.count as f64;
if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) {
props.insert(format!("min_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) {
props.insert(format!("max_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(avg) {
props.insert(format!("avg_{key}"), Value::Number(nm));
}
}
}
}
// Build GeoJSON Feature
let mut feature = Map::new();
feature.insert("type".into(), Value::String("Feature".into()));
@ -241,6 +339,7 @@ pub async fn get_postcodes(
bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east),
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
travel_entries = travel_entries.len(),
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
"GET /api/postcodes"
);

View file

@ -130,7 +130,7 @@ pub fn build_property(
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form",
),
duration: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Leashold/Freehold",
feature_name_to_index, feature_data, num_features, enum_values, row, "Leasehold/Freehold",
),
current_energy_rating: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating",

View file

@ -95,6 +95,13 @@ pub async fn post_shorten(state: Arc<AppState>, Json(req): Json<ShortenRequest>)
}
pub async fn get_short_url(state: Arc<AppState>, Path(code): Path<String>) -> Response {
if code.is_empty()
|| code.len() > 20
|| !code.bytes().all(|b| b.is_ascii_alphanumeric())
{
return StatusCode::BAD_REQUEST.into_response();
}
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(

View file

@ -1,4 +1,62 @@
/// Per-hex-cell travel time aggregation.
/// A parsed travel time entry from the `travel` query parameter.
pub struct TravelEntry {
pub mode: String,
pub slug: String,
pub use_best: bool,
pub filter_min: Option<f32>,
pub filter_max: Option<f32>,
}
/// Parse `travel` param into a list of travel entries.
/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max`
pub fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mut entries = Vec::new();
let mut seen_keys = Vec::new();
for segment in travel_str.split('|') {
let parts: Vec<&str> = segment.split(':').collect();
if parts.len() < 2 {
return Err(format!(
"each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'",
segment
));
}
let mode = parts[0].trim().to_string();
let slug = parts[1].trim().to_string();
let use_best = parts.len() >= 3 && parts[2].trim() == "best";
let filter_offset = if use_best { 1 } else { 0 };
let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset {
let min: f32 = parts[2 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
let max: f32 = parts[3 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
(Some(min), Some(max))
} else {
(None, None)
};
let key = format!("{}:{}", mode, slug);
if seen_keys.contains(&key) {
return Err(format!("duplicate travel entry '{}'", key));
}
seen_keys.push(key);
entries.push(TravelEntry {
mode,
slug,
use_best,
filter_min,
filter_max,
});
}
Ok(entries)
}
/// Per-cell travel time aggregation.
pub struct TravelTimeAgg {
pub min: f32,
pub max: f32,