More fixes
This commit is contained in:
parent
791bc6976b
commit
14a3555cf1
21 changed files with 549 additions and 99 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,6 +1,5 @@
|
||||||
.venv
|
.venv
|
||||||
.claude
|
.claude
|
||||||
tfl_journey_client
|
|
||||||
**/node_modules
|
**/node_modules
|
||||||
**/__pycache__
|
**/__pycache__
|
||||||
**/dist
|
**/dist
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@ PLACES := $(DATA_DIR)/places.parquet
|
||||||
LISTINGS_BUY := $(DATA_DIR)/online_listings_buy.parquet
|
LISTINGS_BUY := $(DATA_DIR)/online_listings_buy.parquet
|
||||||
LISTINGS_RENT := $(DATA_DIR)/online_listings_rent.parquet
|
LISTINGS_RENT := $(DATA_DIR)/online_listings_rent.parquet
|
||||||
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
|
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
|
||||||
|
RM_OUTCODES := frontend/src/lib/rightmove-outcodes.json
|
||||||
|
|
||||||
# Sentinel files for directory targets (Make doesn't track directories well)
|
# Sentinel files for directory targets (Make doesn't track directories well)
|
||||||
GEOSURE_STAMP := $(GEOSURE_DIR)/.done
|
GEOSURE_STAMP := $(GEOSURE_DIR)/.done
|
||||||
|
|
@ -64,7 +65,7 @@ PMTILES_VERSION := 1.22.3
|
||||||
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
||||||
download-naptan download-pois download-ofsted download-broadband download-rental-prices \
|
download-naptan download-pois download-ofsted download-broadband download-rental-prices \
|
||||||
download-postcodes download-geosure download-noise download-inspire \
|
download-postcodes download-geosure download-noise download-inspire \
|
||||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population \
|
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-pbf download-places download-lsoa-population download-rightmove-outcodes \
|
||||||
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
||||||
transform-school-proximity transform-geosure transform-postcode-boundaries \
|
transform-school-proximity transform-geosure transform-postcode-boundaries \
|
||||||
generate-postcode-boundaries
|
generate-postcode-boundaries
|
||||||
|
|
@ -92,6 +93,7 @@ download-greenspace: $(GREENSPACE)
|
||||||
download-pbf: $(PBF)
|
download-pbf: $(PBF)
|
||||||
download-places: $(PLACES)
|
download-places: $(PLACES)
|
||||||
download-lsoa-population: $(LSOA_POP)
|
download-lsoa-population: $(LSOA_POP)
|
||||||
|
download-rightmove-outcodes: $(RM_OUTCODES)
|
||||||
transform-pois: $(POIS_FILTERED)
|
transform-pois: $(POIS_FILTERED)
|
||||||
transform-epc-pp: $(EPC_PP)
|
transform-epc-pp: $(EPC_PP)
|
||||||
transform-crime: $(CRIME)
|
transform-crime: $(CRIME)
|
||||||
|
|
@ -187,6 +189,9 @@ $(PLACES): $(PBF)
|
||||||
$(LSOA_POP):
|
$(LSOA_POP):
|
||||||
uv run python -m pipeline.download.lsoa_population --output $@
|
uv run python -m pipeline.download.lsoa_population --output $@
|
||||||
|
|
||||||
|
$(RM_OUTCODES): $(MERGE_STAMP)
|
||||||
|
uv run python -m pipeline.download.rightmove_outcodes --postcodes $(POSTCODES_PQ) --output $@
|
||||||
|
|
||||||
# ── Transforms ────────────────────────────────────────────────────────────────
|
# ── Transforms ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN)
|
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN)
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,10 @@ SEED = 42
|
||||||
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
||||||
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
||||||
# Whether to run a scrape immediately on startup
|
# Whether to run a scrape immediately on startup
|
||||||
RUN_ON_STARTUP = False
|
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
|
||||||
|
# Enable/disable individual sources
|
||||||
|
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes")
|
||||||
|
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes")
|
||||||
|
|
||||||
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
||||||
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
|
||||||
"Address per Property Register": [
|
"Address per Property Register": [
|
||||||
p["Address per Property Register"] for p in properties
|
p["Address per Property Register"] for p in properties
|
||||||
],
|
],
|
||||||
"Leashold/Freehold": [p["Leashold/Freehold"] for p in properties],
|
"Leasehold/Freehold": [p["Leasehold/Freehold"] for p in properties],
|
||||||
"Property type": [p["Property type"] for p in properties],
|
"Property type": [p["Property type"] for p in properties],
|
||||||
"Property sub-type": [p["Property sub-type"] for p in properties],
|
"Property sub-type": [p["Property sub-type"] for p in properties],
|
||||||
"Price qualifier": [p["Price qualifier"] for p in properties],
|
"Price qualifier": [p["Price qualifier"] for p in properties],
|
||||||
|
|
@ -78,7 +78,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
|
||||||
"lat": pl.Float64,
|
"lat": pl.Float64,
|
||||||
"Postcode": pl.Utf8,
|
"Postcode": pl.Utf8,
|
||||||
"Address per Property Register": pl.Utf8,
|
"Address per Property Register": pl.Utf8,
|
||||||
"Leashold/Freehold": pl.Utf8,
|
"Leasehold/Freehold": pl.Utf8,
|
||||||
"Property type": pl.Utf8,
|
"Property type": pl.Utf8,
|
||||||
"Property sub-type": pl.Utf8,
|
"Property sub-type": pl.Utf8,
|
||||||
"Price qualifier": pl.Utf8,
|
"Price qualifier": pl.Utf8,
|
||||||
|
|
|
||||||
|
|
@ -59,16 +59,6 @@ export default memo(function HoverCard({ x, y, id, isPostcode, data, filters, fe
|
||||||
transform: 'translate(-50%, -100%)',
|
transform: 'translate(-50%, -100%)',
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{/* Arrow */}
|
|
||||||
<div
|
|
||||||
className="absolute w-3 h-3 bg-white dark:bg-warm-800 rotate-45"
|
|
||||||
style={{
|
|
||||||
left: '50%',
|
|
||||||
bottom: -6,
|
|
||||||
transform: 'translateX(-50%)',
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<div className="relative">
|
<div className="relative">
|
||||||
{/* Header */}
|
{/* Header */}
|
||||||
<div className="flex items-center justify-between gap-2 mb-1">
|
<div className="flex items-center justify-between gap-2 mb-1">
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ import type {
|
||||||
} from '../../types';
|
} from '../../types';
|
||||||
|
|
||||||
import { zoomToResolution, getBoundsFromViewState, getMapStyle } from '../../lib/map-utils';
|
import { zoomToResolution, getBoundsFromViewState, getMapStyle } from '../../lib/map-utils';
|
||||||
import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS } from '../../lib/consts';
|
import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS, POI_GROUP_COLORS, POI_DEFAULT_COLOR } from '../../lib/consts';
|
||||||
import LocationSearch, { type SearchedLocation } from './LocationSearch';
|
import LocationSearch, { type SearchedLocation } from './LocationSearch';
|
||||||
import MapLegend from './MapLegend';
|
import MapLegend from './MapLegend';
|
||||||
import HoverCard from './HoverCard';
|
import HoverCard from './HoverCard';
|
||||||
|
|
@ -263,25 +263,49 @@ export default memo(function Map({
|
||||||
))}
|
))}
|
||||||
{popupInfo && (
|
{popupInfo && (
|
||||||
<div
|
<div
|
||||||
className="absolute bg-white dark:bg-warm-800 rounded shadow-lg p-2 text-sm dark:text-white"
|
className="absolute bg-white dark:bg-warm-800 rounded-lg shadow-lg text-sm dark:text-white pointer-events-none"
|
||||||
style={{
|
style={{
|
||||||
left: popupInfo.x,
|
left: popupInfo.x,
|
||||||
top: popupInfo.y - 40,
|
top: popupInfo.y - 50,
|
||||||
transform: 'translateX(-50%)',
|
transform: 'translateX(-50%)',
|
||||||
zIndex: 9999,
|
zIndex: 9999,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<strong className="dark:text-white">{popupInfo.name}</strong>
|
{popupInfo.isCluster ? (
|
||||||
<div className="text-warm-500 dark:text-warm-300 text-xs">{popupInfo.category}</div>
|
<div className="px-3 py-2 text-center">
|
||||||
{osmIdToUrl(popupInfo.id) && (
|
<div className="text-lg font-bold text-teal-600 dark:text-teal-400">
|
||||||
<a
|
{popupInfo.clusterCount}
|
||||||
href={osmIdToUrl(popupInfo.id)!}
|
</div>
|
||||||
target="_blank"
|
<div className="text-warm-500 dark:text-warm-400 text-xs">places</div>
|
||||||
rel="noopener noreferrer"
|
</div>
|
||||||
className="text-teal-600 dark:text-teal-400 hover:text-teal-800 dark:hover:text-teal-300 text-xs"
|
) : (
|
||||||
>
|
<div className="px-3 py-2">
|
||||||
View on OSM
|
<div className="flex items-center gap-2">
|
||||||
</a>
|
<span className="text-lg leading-none">{popupInfo.emoji}</span>
|
||||||
|
<div>
|
||||||
|
<div className="font-semibold dark:text-warm-100">{popupInfo.name}</div>
|
||||||
|
<div className="flex items-center gap-1.5 text-xs text-warm-500 dark:text-warm-400">
|
||||||
|
<span
|
||||||
|
className="inline-block w-2 h-2 rounded-full flex-shrink-0"
|
||||||
|
style={{
|
||||||
|
backgroundColor: `rgb(${(POI_GROUP_COLORS[popupInfo.group] || POI_DEFAULT_COLOR).join(',')})`,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
{popupInfo.category}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{osmIdToUrl(popupInfo.id) && (
|
||||||
|
<a
|
||||||
|
href={osmIdToUrl(popupInfo.id)!}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="text-teal-600 dark:text-teal-400 hover:text-teal-800 dark:hover:text-teal-300 text-xs mt-1 block pointer-events-auto"
|
||||||
|
>
|
||||||
|
View on OSM
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
|
||||||
|
|
@ -135,17 +135,6 @@ function PropertyLoadingSkeleton() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const LISTING_STATUS_STYLES: Record<string, string> = {
|
|
||||||
'For sale': 'bg-teal-100 text-teal-800 dark:bg-teal-900/40 dark:text-teal-300',
|
|
||||||
'For rent': 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300',
|
|
||||||
'Historical sale': 'bg-warm-100 text-warm-600 dark:bg-warm-700 dark:text-warm-300',
|
|
||||||
};
|
|
||||||
|
|
||||||
function ListingStatusBadge({ status }: { status: string }) {
|
|
||||||
const style = LISTING_STATUS_STYLES[status] ?? LISTING_STATUS_STYLES['Historical sale'];
|
|
||||||
return <span className={`text-xs font-medium px-1.5 py-0.5 rounded ${style}`}>{status}</span>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function PropertyCard({ property }: { property: Property }) {
|
function PropertyCard({ property }: { property: Property }) {
|
||||||
const price = getNum(property, 'Last known price');
|
const price = getNum(property, 'Last known price');
|
||||||
const estimatedPrice = getNum(property, 'Estimated current price');
|
const estimatedPrice = getNum(property, 'Estimated current price');
|
||||||
|
|
@ -161,18 +150,13 @@ function PropertyCard({ property }: { property: Property }) {
|
||||||
const bathrooms = getNum(property, 'Bathrooms');
|
const bathrooms = getNum(property, 'Bathrooms');
|
||||||
const listingDate = getNum(property, 'Listing date');
|
const listingDate = getNum(property, 'Listing date');
|
||||||
|
|
||||||
const listingStatus = property.listing_status;
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="p-4 border-b border-warm-100 dark:border-warm-800 hover:bg-warm-50 dark:hover:bg-warm-800">
|
<div className="p-4 border-b border-warm-100 dark:border-warm-800 hover:bg-warm-50 dark:hover:bg-warm-800">
|
||||||
<div className="flex items-start justify-between gap-2">
|
<div>
|
||||||
<div>
|
<div className="font-semibold dark:text-warm-100">
|
||||||
<div className="font-semibold dark:text-warm-100">
|
{property.address || 'Unknown Address'}
|
||||||
{property.address || 'Unknown Address'}
|
|
||||||
</div>
|
|
||||||
<div className="text-sm text-warm-600 dark:text-warm-400">{property.postcode}</div>
|
|
||||||
</div>
|
</div>
|
||||||
{listingStatus && <ListingStatusBadge status={listingStatus} />}
|
<div className="text-sm text-warm-600 dark:text-warm-400">{property.postcode}</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{property.property_sub_type && (
|
{property.property_sub_type && (
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
import { useCallback, useRef, useState, useMemo, useEffect } from 'react';
|
import { useCallback, useRef, useState, useMemo, useEffect } from 'react';
|
||||||
import { H3HexagonLayer } from '@deck.gl/geo-layers';
|
import { H3HexagonLayer } from '@deck.gl/geo-layers';
|
||||||
import { GeoJsonLayer, IconLayer, TextLayer } from '@deck.gl/layers';
|
import { GeoJsonLayer, IconLayer, TextLayer, ScatterplotLayer } from '@deck.gl/layers';
|
||||||
import { cellToBoundary } from 'h3-js';
|
import { cellToBoundary } from 'h3-js';
|
||||||
|
import Supercluster from 'supercluster';
|
||||||
import type { PickingInfo } from '@deck.gl/core';
|
import type { PickingInfo } from '@deck.gl/core';
|
||||||
import type {
|
import type {
|
||||||
HexagonData,
|
HexagonData,
|
||||||
|
|
@ -12,7 +13,16 @@ import type {
|
||||||
FeatureMeta,
|
FeatureMeta,
|
||||||
Bounds,
|
Bounds,
|
||||||
} from '../types';
|
} from '../types';
|
||||||
import { DENSITY_GRADIENT, DENSITY_GRADIENT_DARK } from '../lib/consts';
|
import {
|
||||||
|
DENSITY_GRADIENT,
|
||||||
|
DENSITY_GRADIENT_DARK,
|
||||||
|
POI_GROUP_COLORS,
|
||||||
|
POI_DEFAULT_COLOR,
|
||||||
|
MINOR_POI_CATEGORIES,
|
||||||
|
MINOR_POI_ZOOM_THRESHOLD,
|
||||||
|
POI_CLUSTER_RADIUS,
|
||||||
|
POI_CLUSTER_MAX_ZOOM,
|
||||||
|
} from '../lib/consts';
|
||||||
import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils';
|
import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils';
|
||||||
import {
|
import {
|
||||||
type TravelTimeEntry,
|
type TravelTimeEntry,
|
||||||
|
|
@ -55,7 +65,18 @@ interface PopupInfo {
|
||||||
y: number;
|
y: number;
|
||||||
name: string;
|
name: string;
|
||||||
category: string;
|
category: string;
|
||||||
|
group: string;
|
||||||
|
emoji: string;
|
||||||
id: string;
|
id: string;
|
||||||
|
isCluster?: boolean;
|
||||||
|
clusterCount?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ClusterPoint {
|
||||||
|
lng: number;
|
||||||
|
lat: number;
|
||||||
|
count: number;
|
||||||
|
clusterId: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useDeckLayers({
|
export function useDeckLayers({
|
||||||
|
|
@ -204,6 +225,8 @@ export function useDeckLayers({
|
||||||
y: info.y,
|
y: info.y,
|
||||||
name: info.object.name,
|
name: info.object.name,
|
||||||
category: info.object.category,
|
category: info.object.category,
|
||||||
|
group: info.object.group,
|
||||||
|
emoji: info.object.emoji,
|
||||||
id: info.object.id,
|
id: info.object.id,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -217,6 +240,30 @@ export function useDeckLayers({
|
||||||
handlePoiHoverRef.current(info);
|
handlePoiHoverRef.current(info);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const handleClusterHover = useCallback((info: PickingInfo<ClusterPoint>) => {
|
||||||
|
if (info.object && info.x !== undefined && info.y !== undefined) {
|
||||||
|
setPopupInfo({
|
||||||
|
x: info.x,
|
||||||
|
y: info.y,
|
||||||
|
name: `${info.object.count} places`,
|
||||||
|
category: 'Zoom in to see details',
|
||||||
|
group: '',
|
||||||
|
emoji: '',
|
||||||
|
id: '',
|
||||||
|
isCluster: true,
|
||||||
|
clusterCount: info.object.count,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
setPopupInfo(null);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleClusterHoverRef = useRef(handleClusterHover);
|
||||||
|
handleClusterHoverRef.current = handleClusterHover;
|
||||||
|
const stableClusterHover = useCallback((info: PickingInfo<ClusterPoint>) => {
|
||||||
|
handleClusterHoverRef.current(info);
|
||||||
|
}, []);
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const handlePostcodeClick = useCallback((info: PickingInfo<any>) => {
|
const handlePostcodeClick = useCallback((info: PickingInfo<any>) => {
|
||||||
const pc = info.object?.properties?.postcode;
|
const pc = info.object?.properties?.postcode;
|
||||||
|
|
@ -461,24 +508,147 @@ export function useDeckLayers({
|
||||||
[postcodeData, theme]
|
[postcodeData, theme]
|
||||||
);
|
);
|
||||||
|
|
||||||
const poiLayer = useMemo(
|
// --- POI clustering ---
|
||||||
|
const clusterIndex = useMemo(() => {
|
||||||
|
if (pois.length === 0) return null;
|
||||||
|
const index = new Supercluster<POI>({
|
||||||
|
radius: POI_CLUSTER_RADIUS,
|
||||||
|
maxZoom: POI_CLUSTER_MAX_ZOOM,
|
||||||
|
});
|
||||||
|
const features: Supercluster.PointFeature<POI>[] = pois.map((poi) => ({
|
||||||
|
type: 'Feature',
|
||||||
|
geometry: { type: 'Point', coordinates: [poi.lng, poi.lat] },
|
||||||
|
properties: poi,
|
||||||
|
}));
|
||||||
|
index.load(features);
|
||||||
|
return index;
|
||||||
|
}, [pois]);
|
||||||
|
|
||||||
|
const clusterZoom = Math.floor(zoom);
|
||||||
|
const showMinorPois = zoom >= MINOR_POI_ZOOM_THRESHOLD;
|
||||||
|
|
||||||
|
const { visiblePois, clusters } = useMemo(() => {
|
||||||
|
if (!clusterIndex || pois.length === 0) {
|
||||||
|
return { visiblePois: [] as POI[], clusters: [] as ClusterPoint[] };
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
const allFeatures = clusterIndex.getClusters([-180, -85, 180, 85], clusterZoom) as any[];
|
||||||
|
const individual: POI[] = [];
|
||||||
|
const clusterPoints: ClusterPoint[] = [];
|
||||||
|
for (const feature of allFeatures) {
|
||||||
|
if (feature.properties.cluster) {
|
||||||
|
clusterPoints.push({
|
||||||
|
lng: feature.geometry.coordinates[0],
|
||||||
|
lat: feature.geometry.coordinates[1],
|
||||||
|
count: feature.properties.point_count,
|
||||||
|
clusterId: feature.properties.cluster_id,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
const poi = feature.properties as POI;
|
||||||
|
if (!showMinorPois && MINOR_POI_CATEGORIES.has(poi.category)) continue;
|
||||||
|
individual.push(poi);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { visiblePois: individual, clusters: clusterPoints };
|
||||||
|
}, [clusterIndex, clusterZoom, showMinorPois, pois]);
|
||||||
|
|
||||||
|
// --- Individual POI layers (shadow → background → emoji) ---
|
||||||
|
const poiShadowLayer = useMemo(
|
||||||
|
() =>
|
||||||
|
new ScatterplotLayer<POI>({
|
||||||
|
id: 'poi-shadow',
|
||||||
|
data: visiblePois,
|
||||||
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
|
getRadius: 16,
|
||||||
|
radiusUnits: 'pixels',
|
||||||
|
getFillColor: isDark ? [0, 0, 0, 50] : [0, 0, 0, 25],
|
||||||
|
pickable: false,
|
||||||
|
transitions: { getRadius: { duration: 300, enter: () => [0] } },
|
||||||
|
}),
|
||||||
|
[visiblePois, isDark]
|
||||||
|
);
|
||||||
|
|
||||||
|
const poiBackgroundLayer = useMemo(
|
||||||
|
() =>
|
||||||
|
new ScatterplotLayer<POI>({
|
||||||
|
id: 'poi-background',
|
||||||
|
data: visiblePois,
|
||||||
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
|
getRadius: 14,
|
||||||
|
radiusUnits: 'pixels',
|
||||||
|
getFillColor: isDark ? [41, 37, 36, 255] : [255, 255, 255, 255],
|
||||||
|
getLineColor: (d) => {
|
||||||
|
const c = POI_GROUP_COLORS[d.group] || POI_DEFAULT_COLOR;
|
||||||
|
return [c[0], c[1], c[2], 255] as [number, number, number, number];
|
||||||
|
},
|
||||||
|
getLineWidth: 2.5,
|
||||||
|
lineWidthUnits: 'pixels',
|
||||||
|
stroked: true,
|
||||||
|
pickable: true,
|
||||||
|
onHover: stablePoiHover,
|
||||||
|
transitions: { getRadius: { duration: 300, enter: () => [0] } },
|
||||||
|
}),
|
||||||
|
[visiblePois, isDark, stablePoiHover]
|
||||||
|
);
|
||||||
|
|
||||||
|
const poiIconLayer = useMemo(
|
||||||
() =>
|
() =>
|
||||||
new IconLayer<POI>({
|
new IconLayer<POI>({
|
||||||
id: 'poi-icons',
|
id: 'poi-icons',
|
||||||
data: pois,
|
data: visiblePois,
|
||||||
getPosition: (d) => [d.lng, d.lat],
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
getIcon: (d) => ({
|
getIcon: (d) => ({
|
||||||
url: emojiToTwemojiUrl(d.emoji),
|
url: emojiToTwemojiUrl(d.emoji),
|
||||||
width: 72,
|
width: 72,
|
||||||
height: 72,
|
height: 72,
|
||||||
}),
|
}),
|
||||||
getSize: 24,
|
getSize: 18,
|
||||||
sizeMinPixels: 20,
|
sizeUnits: 'pixels',
|
||||||
sizeMaxPixels: 40,
|
pickable: false,
|
||||||
pickable: true,
|
transitions: { getSize: { duration: 300, enter: () => [0] } },
|
||||||
onHover: stablePoiHover,
|
|
||||||
}),
|
}),
|
||||||
[pois, stablePoiHover]
|
[visiblePois]
|
||||||
|
);
|
||||||
|
|
||||||
|
// --- Cluster layers ---
|
||||||
|
const clusterCircleLayer = useMemo(
|
||||||
|
() =>
|
||||||
|
new ScatterplotLayer<ClusterPoint>({
|
||||||
|
id: 'poi-clusters',
|
||||||
|
data: clusters,
|
||||||
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
|
getRadius: (d) => Math.min(30, 14 + Math.sqrt(d.count) * 2),
|
||||||
|
radiusUnits: 'pixels',
|
||||||
|
getFillColor: isDark ? [5, 129, 114, 220] : [20, 184, 166, 220],
|
||||||
|
getLineColor: [255, 255, 255, isDark ? 60 : 120],
|
||||||
|
getLineWidth: 2,
|
||||||
|
lineWidthUnits: 'pixels',
|
||||||
|
stroked: true,
|
||||||
|
pickable: true,
|
||||||
|
onHover: stableClusterHover,
|
||||||
|
transitions: { getRadius: { duration: 300, enter: () => [0] } },
|
||||||
|
}),
|
||||||
|
[clusters, isDark, stableClusterHover]
|
||||||
|
);
|
||||||
|
|
||||||
|
const clusterTextLayer = useMemo(
|
||||||
|
() =>
|
||||||
|
new TextLayer<ClusterPoint>({
|
||||||
|
id: 'poi-cluster-text',
|
||||||
|
data: clusters,
|
||||||
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
|
getText: (d) =>
|
||||||
|
d.count >= 1000 ? `${(d.count / 1000).toFixed(1)}k` : String(d.count),
|
||||||
|
getSize: 12,
|
||||||
|
getColor: [255, 255, 255, 255],
|
||||||
|
fontWeight: 700,
|
||||||
|
fontFamily: 'Inter, system-ui, sans-serif',
|
||||||
|
getTextAnchor: 'middle',
|
||||||
|
getAlignmentBaseline: 'center',
|
||||||
|
sizeUnits: 'pixels',
|
||||||
|
pickable: false,
|
||||||
|
}),
|
||||||
|
[clusters]
|
||||||
);
|
);
|
||||||
|
|
||||||
// Marching ants highlight layer for selected hexagon or postcode
|
// Marching ants highlight layer for selected hexagon or postcode
|
||||||
|
|
@ -511,13 +681,18 @@ export function useDeckLayers({
|
||||||
});
|
});
|
||||||
}, [selectedPostcodeGeometry, selectedHexagonId, marchTime]);
|
}, [selectedPostcodeGeometry, selectedHexagonId, marchTime]);
|
||||||
|
|
||||||
|
const poiLayers = useMemo(
|
||||||
|
() => [poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer],
|
||||||
|
[poiShadowLayer, poiBackgroundLayer, poiIconLayer, clusterCircleLayer, clusterTextLayer]
|
||||||
|
);
|
||||||
|
|
||||||
const layers = useMemo(() => {
|
const layers = useMemo(() => {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const baseLayers: any[] = usePostcodeView
|
const baseLayers: any[] = usePostcodeView
|
||||||
? zoom >= 16
|
? zoom >= 16
|
||||||
? [postcodeLayer, postcodeLabelsLayer, poiLayer]
|
? [postcodeLayer, postcodeLabelsLayer, ...poiLayers]
|
||||||
: [postcodeLayer, poiLayer]
|
: [postcodeLayer, ...poiLayers]
|
||||||
: [hexLayer, poiLayer];
|
: [hexLayer, ...poiLayers];
|
||||||
if (marchingAntsLayer) baseLayers.push(marchingAntsLayer);
|
if (marchingAntsLayer) baseLayers.push(marchingAntsLayer);
|
||||||
return baseLayers;
|
return baseLayers;
|
||||||
}, [
|
}, [
|
||||||
|
|
@ -526,7 +701,7 @@ export function useDeckLayers({
|
||||||
hexLayer,
|
hexLayer,
|
||||||
postcodeLayer,
|
postcodeLayer,
|
||||||
postcodeLabelsLayer,
|
postcodeLabelsLayer,
|
||||||
poiLayer,
|
poiLayers,
|
||||||
marchingAntsLayer,
|
marchingAntsLayer,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,13 +16,20 @@ interface SelectedHexagon {
|
||||||
resolution: number;
|
resolution: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface JourneyDest {
|
||||||
|
mode: string;
|
||||||
|
slug: string;
|
||||||
|
}
|
||||||
|
|
||||||
interface UseHexagonSelectionOptions {
|
interface UseHexagonSelectionOptions {
|
||||||
filters: FeatureFilters;
|
filters: FeatureFilters;
|
||||||
features: FeatureMeta[];
|
features: FeatureMeta[];
|
||||||
resolution: number;
|
resolution: number;
|
||||||
|
/** First transit destination — used to pick the best central_postcode for journey display. */
|
||||||
|
journeyDest?: JourneyDest | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useHexagonSelection({ filters, features, resolution }: UseHexagonSelectionOptions) {
|
export function useHexagonSelection({ filters, features, resolution, journeyDest }: UseHexagonSelectionOptions) {
|
||||||
const [selectedHexagon, setSelectedHexagon] = useState<SelectedHexagon | null>(null);
|
const [selectedHexagon, setSelectedHexagon] = useState<SelectedHexagon | null>(null);
|
||||||
const [properties, setProperties] = useState<Property[]>([]);
|
const [properties, setProperties] = useState<Property[]>([]);
|
||||||
const [propertiesTotal, setPropertiesTotal] = useState(0);
|
const [propertiesTotal, setPropertiesTotal] = useState(0);
|
||||||
|
|
@ -46,11 +53,15 @@ export function useHexagonSelection({ filters, features, resolution }: UseHexago
|
||||||
if (fields) {
|
if (fields) {
|
||||||
params.set('fields', fields.join(','));
|
params.set('fields', fields.join(','));
|
||||||
}
|
}
|
||||||
|
if (journeyDest) {
|
||||||
|
params.set('journey_mode', journeyDest.mode);
|
||||||
|
params.set('journey_slug', journeyDest.slug);
|
||||||
|
}
|
||||||
const response = await fetch(apiUrl('hexagon-stats', params), authHeaders({ signal }));
|
const response = await fetch(apiUrl('hexagon-stats', params), authHeaders({ signal }));
|
||||||
assertOk(response, 'hexagon-stats');
|
assertOk(response, 'hexagon-stats');
|
||||||
return (await response.json()) as HexagonStatsResponse;
|
return (await response.json()) as HexagonStatsResponse;
|
||||||
},
|
},
|
||||||
[filters, features]
|
[filters, features, journeyDest]
|
||||||
);
|
);
|
||||||
|
|
||||||
const fetchPostcodeStats = useCallback(
|
const fetchPostcodeStats = useCallback(
|
||||||
|
|
|
||||||
46
frontend/src/hooks/useTravelDestinations.ts
Normal file
46
frontend/src/hooks/useTravelDestinations.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
import { useState, useEffect, useRef } from 'react';
|
||||||
|
import { authHeaders, logNonAbortError } from '../lib/api';
|
||||||
|
import type { TransportMode } from './useTravelTime';
|
||||||
|
|
||||||
|
export interface Destination {
|
||||||
|
name: string;
|
||||||
|
slug: string;
|
||||||
|
place_type: string;
|
||||||
|
city?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fetches all travel-time destinations for a mode once, with client-side caching. */
|
||||||
|
export function useTravelDestinations(mode: TransportMode) {
|
||||||
|
const [destinations, setDestinations] = useState<Destination[]>([]);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const cacheRef = useRef<Partial<Record<TransportMode, Destination[]>>>({});
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (cacheRef.current[mode]) {
|
||||||
|
setDestinations(cacheRef.current[mode]!);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
setLoading(true);
|
||||||
|
|
||||||
|
fetch(
|
||||||
|
`/api/travel-destinations?mode=${mode}`,
|
||||||
|
authHeaders({ signal: controller.signal }),
|
||||||
|
)
|
||||||
|
.then((res) => {
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.then((data: { destinations: Destination[] }) => {
|
||||||
|
cacheRef.current[mode] = data.destinations;
|
||||||
|
setDestinations(data.destinations);
|
||||||
|
})
|
||||||
|
.catch((err) => logNonAbortError('travel destinations', err))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
|
||||||
|
return () => controller.abort();
|
||||||
|
}, [mode]);
|
||||||
|
|
||||||
|
return { destinations, loading };
|
||||||
|
}
|
||||||
|
|
@ -47,7 +47,7 @@ export function useTravelTime(initial?: TravelTimeInitial) {
|
||||||
(index: number, slug: string, label: string) => {
|
(index: number, slug: string, label: string) => {
|
||||||
setEntries((prev) =>
|
setEntries((prev) =>
|
||||||
prev.map((entry, i) =>
|
prev.map((entry, i) =>
|
||||||
i === index ? { ...entry, slug, label, timeRange: null } : entry
|
i === index ? { ...entry, slug, label, timeRange: slug ? [0, 120] : null } : entry
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -28,8 +28,8 @@ export const INITIAL_VIEW_STATE: ViewState = {
|
||||||
* Returns the H3 resolution to use for a given zoom level.
|
* Returns the H3 resolution to use for a given zoom level.
|
||||||
*/
|
*/
|
||||||
export const ZOOM_TO_RESOLUTION_THRESHOLDS = [
|
export const ZOOM_TO_RESOLUTION_THRESHOLDS = [
|
||||||
{ maxZoom: 7.5, resolution: 5 },
|
{ maxZoom: 7, resolution: 5 },
|
||||||
{ maxZoom: 9.5, resolution: 6 },
|
{ maxZoom: 9, resolution: 6 },
|
||||||
{ maxZoom: 10.5, resolution: 7 },
|
{ maxZoom: 10.5, resolution: 7 },
|
||||||
{ maxZoom: 11.5, resolution: 8 },
|
{ maxZoom: 11.5, resolution: 8 },
|
||||||
{ maxZoom: 13, resolution: 9 },
|
{ maxZoom: 13, resolution: 9 },
|
||||||
|
|
@ -68,6 +68,41 @@ export const GLYPHS_URL = '/assets/fonts/{fontstack}/{range}.pbf';
|
||||||
/** Twemoji base URL (served locally from public/assets/) */
|
/** Twemoji base URL (served locally from public/assets/) */
|
||||||
export const TWEMOJI_BASE = '/assets/twemoji/';
|
export const TWEMOJI_BASE = '/assets/twemoji/';
|
||||||
|
|
||||||
|
/** POI group → RGB color for category-coded map markers */
|
||||||
|
export const POI_GROUP_COLORS: Record<string, [number, number, number]> = {
|
||||||
|
'Public Transport': [59, 130, 246],
|
||||||
|
Leisure: [249, 115, 22],
|
||||||
|
Education: [139, 92, 246],
|
||||||
|
Health: [239, 68, 68],
|
||||||
|
'Emergency Services': [220, 38, 38],
|
||||||
|
Other: [107, 114, 128],
|
||||||
|
Groceries: [34, 197, 94],
|
||||||
|
'Local Businesses': [245, 158, 11],
|
||||||
|
Culture: [236, 72, 153],
|
||||||
|
Services: [6, 182, 212],
|
||||||
|
Shops: [99, 102, 241],
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Default color for unknown POI groups */
|
||||||
|
export const POI_DEFAULT_COLOR: [number, number, number] = [107, 114, 128];
|
||||||
|
|
||||||
|
/** Categories only shown when zoomed in past MINOR_POI_ZOOM_THRESHOLD */
|
||||||
|
export const MINOR_POI_CATEGORIES = new Set([
|
||||||
|
'Bus stop',
|
||||||
|
'Taxi rank',
|
||||||
|
'EV Charging',
|
||||||
|
'Playground',
|
||||||
|
]);
|
||||||
|
|
||||||
|
/** Zoom level below which minor POI categories are hidden */
|
||||||
|
export const MINOR_POI_ZOOM_THRESHOLD = 14;
|
||||||
|
|
||||||
|
/** Supercluster grouping radius in pixels */
|
||||||
|
export const POI_CLUSTER_RADIUS = 50;
|
||||||
|
|
||||||
|
/** Zoom level at which supercluster stops clustering */
|
||||||
|
export const POI_CLUSTER_MAX_ZOOM = 15;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Groups whose features should be collapsed into stacked bar charts.
|
* Groups whose features should be collapsed into stacked bar charts.
|
||||||
* Keyed by feature group name. Each entry defines one stacked chart.
|
* Keyed by feature group name. Each entry defines one stacked chart.
|
||||||
|
|
@ -153,8 +188,8 @@ export const STACKED_ENUM_GROUPS: Record<
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: 'Leasehold/Freehold',
|
label: 'Leasehold/Freehold',
|
||||||
feature: 'Leasehold/Freehold',
|
feature: 'Leashold/Freehold',
|
||||||
components: ['Leasehold/Freehold'],
|
components: ['Leashold/Freehold'],
|
||||||
valueOrder: ['Freehold', 'Leasehold'],
|
valueOrder: ['Freehold', 'Leasehold'],
|
||||||
valueColors: ['#3b82f6', '#f59e0b'],
|
valueColors: ['#3b82f6', '#f59e0b'],
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,29 @@
|
||||||
"""Compute POI proximity counts per postcode from ArcGIS + filtered POIs."""
|
"""Compute POI proximity counts and distances per postcode from ArcGIS + filtered POIs."""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import polars as pl
|
import polars as pl
|
||||||
|
|
||||||
from pipeline.utils.poi_counts import count_pois_per_postcode
|
from pipeline.utils.poi_counts import count_pois_per_postcode, min_distance_per_postcode
|
||||||
|
|
||||||
|
|
||||||
# POI category groups for proximity counting.
|
# POI category groups for proximity counting (2km radius).
|
||||||
# Names must match the friendly names produced by transform_poi.py / naptan.py.
|
# Names must match the friendly names produced by transform_poi.py / naptan.py.
|
||||||
POI_GROUPS = {
|
POI_GROUPS_2KM = {
|
||||||
"restaurants": ["Restaurant", "Fast Food"],
|
"restaurants": ["Restaurant", "Fast Food"],
|
||||||
"groceries": ["Greengrocer", "Supermarket", "Convenience Store"],
|
"groceries": ["Greengrocer", "Supermarket", "Convenience Store"],
|
||||||
"parks": ["Park"],
|
"parks": ["Park"],
|
||||||
"public_transport": [
|
}
|
||||||
"Metro or Tram stop",
|
|
||||||
"Rail station",
|
# Train/tube stations counted at 1km radius
|
||||||
"Bus stop",
|
TRAIN_TUBE_GROUP = {
|
||||||
"Bus station",
|
"train_tube": ["Metro or Tram stop", "Rail station"],
|
||||||
], # comes from naptan.py
|
}
|
||||||
|
|
||||||
|
# Groups for which to compute distance to nearest POI
|
||||||
|
DISTANCE_GROUPS = {
|
||||||
|
"train_tube": ["Metro or Tram stop", "Rail station"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -46,7 +50,21 @@ def main():
|
||||||
|
|
||||||
pois = pl.read_parquet(args.pois)
|
pois = pl.read_parquet(args.pois)
|
||||||
|
|
||||||
result = count_pois_per_postcode(postcodes, pois, groups=POI_GROUPS, radius_km=2)
|
# Count amenity POIs within 2km
|
||||||
|
counts_2km = count_pois_per_postcode(
|
||||||
|
postcodes, pois, groups=POI_GROUPS_2KM, radius_km=2
|
||||||
|
)
|
||||||
|
|
||||||
|
# Count train/tube stations within 1km
|
||||||
|
counts_1km = count_pois_per_postcode(
|
||||||
|
postcodes, pois, groups=TRAIN_TUBE_GROUP, radius_km=1
|
||||||
|
)
|
||||||
|
|
||||||
|
# Distance to nearest train/tube station
|
||||||
|
distances = min_distance_per_postcode(postcodes, pois, groups=DISTANCE_GROUPS)
|
||||||
|
|
||||||
|
# Join all results on postcode
|
||||||
|
result = counts_2km.join(counts_1km, on="postcode").join(distances, on="postcode")
|
||||||
|
|
||||||
result.write_parquet(args.output)
|
result.write_parquet(args.output)
|
||||||
size_mb = args.output.stat().st_size / (1024 * 1024)
|
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||||
|
|
|
||||||
|
|
@ -16,8 +16,8 @@ def compute_voronoi_regions(
|
||||||
if len(points) == 1:
|
if len(points) == 1:
|
||||||
return {postcodes[0]: boundary}
|
return {postcodes[0]: boundary}
|
||||||
|
|
||||||
# UPRN coordinates are int64 (BNG grid refs in whole meters).
|
# UPRN coordinates are int64 (BNG grid refs in whole metres).
|
||||||
# Convert to float64 so sub-meter jitter isn't truncated.
|
# Convert to float64 so sub-metre jitter isn't truncated.
|
||||||
points = points.astype(np.float64)
|
points = points.astype(np.float64)
|
||||||
|
|
||||||
# Deduplicate points, keeping one per (location, postcode) pair.
|
# Deduplicate points, keeping one per (location, postcode) pair.
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ pub const H3_REQUEST_MAX: u8 = 12;
|
||||||
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
|
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
|
||||||
|
|
||||||
pub const GRID_CELL_SIZE: f32 = 0.01;
|
pub const GRID_CELL_SIZE: f32 = 0.01;
|
||||||
pub const MAX_POIS_PER_REQUEST: usize = 2500;
|
pub const MAX_POIS_PER_REQUEST: usize = 10000;
|
||||||
pub const MAX_CELLS_PER_REQUEST: usize = 5000;
|
pub const MAX_CELLS_PER_REQUEST: usize = 5000;
|
||||||
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
|
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
|
||||||
pub const MAX_PROPERTIES_LIMIT: usize = 500;
|
pub const MAX_PROPERTIES_LIMIT: usize = 500;
|
||||||
|
|
@ -24,8 +24,9 @@ pub const SERVICE_CALL_TIMEOUT: u64 = 120;
|
||||||
/// Users without a license can only query data within these bounds.
|
/// Users without a license can only query data within these bounds.
|
||||||
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.42, -0.34, 51.60, 0.14);
|
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.42, -0.34, 51.60, 0.14);
|
||||||
|
|
||||||
/// Homepage demo center (lat, lng). Unlicensed hexagon requests are allowed
|
/// Homepage demo center (lat, lng) and tolerance for the license bypass.
|
||||||
/// when the center of the requested bounds is within DEMO_CENTER_TOLERANCE of this point.
|
/// Hexagon requests centered within this tolerance skip the license check,
|
||||||
/// Must match DEMO_VIEW_START in ScrollStory.tsx.
|
/// so the ScrollStory animation works for anonymous visitors.
|
||||||
|
/// ~0.05° ≈ 5.5 km — covers central London only.
|
||||||
pub const DEMO_CENTER: (f64, f64) = (51.51, -0.12);
|
pub const DEMO_CENTER: (f64, f64) = (51.51, -0.12);
|
||||||
pub const DEMO_CENTER_TOLERANCE: f64 = 1.0;
|
pub const DEMO_CENTER_TOLERANCE: f64 = 0.05;
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use std::collections::VecDeque;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{bail, Context};
|
use anyhow::Context;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use polars::lazy::frame::LazyFrame;
|
use polars::lazy::frame::LazyFrame;
|
||||||
use rustc_hash::{FxHashMap, FxHashSet};
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
|
|
@ -167,19 +167,16 @@ impl TravelTimeStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve slug to actual filename (may have numeric prefix)
|
// Resolve slug to actual filename (may have numeric prefix).
|
||||||
|
// Reject unknown slugs rather than falling back to raw input to prevent path traversal.
|
||||||
let file_stem = self
|
let file_stem = self
|
||||||
.slug_to_file
|
.slug_to_file
|
||||||
.get(&key)
|
.get(&key)
|
||||||
.map(|val| val.as_str())
|
.ok_or_else(|| anyhow::anyhow!("Unknown travel destination: {mode}/{slug}"))?;
|
||||||
.unwrap_or(slug);
|
|
||||||
let path = self
|
let path = self
|
||||||
.base_dir
|
.base_dir
|
||||||
.join(mode)
|
.join(mode)
|
||||||
.join(format!("{}.parquet", file_stem));
|
.join(format!("{}.parquet", file_stem));
|
||||||
if !path.exists() {
|
|
||||||
bail!("Travel time file not found: {}", path.display());
|
|
||||||
}
|
|
||||||
|
|
||||||
let df = LazyFrame::scan_parquet(&path, Default::default())
|
let df = LazyFrame::scan_parquet(&path, Default::default())
|
||||||
.with_context(|| format!("Failed to scan: {}", path.display()))?
|
.with_context(|| format!("Failed to scan: {}", path.display()))?
|
||||||
|
|
|
||||||
|
|
@ -139,8 +139,6 @@ pub async fn get_hexagons(
|
||||||
let (south, west, north, east) =
|
let (south, west, north, east) =
|
||||||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||||
|
|
||||||
// Allow the homepage demo: check if the center of the requested bounds
|
|
||||||
// is near the demo view center (51.51, -0.12).
|
|
||||||
let center_lat = (south + north) / 2.0;
|
let center_lat = (south + north) / 2.0;
|
||||||
let center_lng = (west + east) / 2.0;
|
let center_lng = (west + east) / 2.0;
|
||||||
let is_demo_view = (center_lat - DEMO_CENTER.0).abs() <= DEMO_CENTER_TOLERANCE
|
let is_demo_view = (center_lat - DEMO_CENTER.0).abs() <= DEMO_CENTER_TOLERANCE
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,12 @@ use tracing::info;
|
||||||
use crate::aggregation::Aggregator;
|
use crate::aggregation::Aggregator;
|
||||||
use crate::auth::OptionalUser;
|
use crate::auth::OptionalUser;
|
||||||
use crate::consts::MAX_CELLS_PER_REQUEST;
|
use crate::consts::MAX_CELLS_PER_REQUEST;
|
||||||
|
use crate::data::travel_time::TravelData;
|
||||||
use crate::licensing::check_license_bounds;
|
use crate::licensing::check_license_bounds;
|
||||||
use crate::parsing::{
|
use crate::parsing::{
|
||||||
bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters,
|
bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters,
|
||||||
};
|
};
|
||||||
|
use crate::routes::travel_time::{parse_travel_entries, TravelTimeAgg};
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -31,6 +33,8 @@ pub struct PostcodeParams {
|
||||||
filters: Option<String>,
|
filters: Option<String>,
|
||||||
/// Comma-separated feature names to include in min/max aggregation.
|
/// Comma-separated feature names to include in min/max aggregation.
|
||||||
fields: Option<String>,
|
fields: Option<String>,
|
||||||
|
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
|
||||||
|
travel: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a GeoJSON geometry object from postcode polygon rings.
|
/// Build a GeoJSON geometry object from postcode polygon rings.
|
||||||
|
|
@ -84,10 +88,41 @@ pub async fn get_postcodes(
|
||||||
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index)
|
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index)
|
||||||
.map_err(|err| (err.0, err.1).into_response())?;
|
.map_err(|err| (err.0, err.1).into_response())?;
|
||||||
|
|
||||||
|
// Parse travel entries
|
||||||
|
let travel_entries = params
|
||||||
|
.travel
|
||||||
|
.as_deref()
|
||||||
|
.filter(|val| !val.is_empty())
|
||||||
|
.map(parse_travel_entries)
|
||||||
|
.transpose()
|
||||||
|
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
let response = tokio::task::spawn_blocking(move || -> Result<PostcodesResponse, String> {
|
let response = tokio::task::spawn_blocking(move || -> Result<PostcodesResponse, String> {
|
||||||
let postcode_data = &state.postcode_data;
|
let postcode_data = &state.postcode_data;
|
||||||
let t0 = std::time::Instant::now();
|
let t0 = std::time::Instant::now();
|
||||||
|
|
||||||
|
// Load travel time data from precomputed parquet files
|
||||||
|
let travel_data: Vec<TravelData> = if !travel_entries.is_empty() {
|
||||||
|
let store = &state.travel_time_store;
|
||||||
|
travel_entries
|
||||||
|
.iter()
|
||||||
|
.map(|entry| {
|
||||||
|
store
|
||||||
|
.get(&entry.mode, &entry.slug)
|
||||||
|
.map_err(|err| format!("Failed to load travel data: {}", err))
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>, _>>()?
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
let has_travel = !travel_entries.is_empty();
|
||||||
|
let travel_field_keys: Vec<String> = travel_entries
|
||||||
|
.iter()
|
||||||
|
.map(|te| format!("tt_{}_{}", te.mode, te.slug))
|
||||||
|
.collect();
|
||||||
|
|
||||||
let num_features = state.data.num_features;
|
let num_features = state.data.num_features;
|
||||||
let feature_data = &state.data.feature_data;
|
let feature_data = &state.data.feature_data;
|
||||||
let min_keys = &state.min_keys;
|
let min_keys = &state.min_keys;
|
||||||
|
|
@ -122,9 +157,35 @@ pub async fn get_postcodes(
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Filter postcodes by travel time range (if specified)
|
||||||
|
if has_travel {
|
||||||
|
postcode_rows.retain(|&pc_idx, _rows| {
|
||||||
|
let postcode = &postcode_data.postcodes[pc_idx];
|
||||||
|
for (ti, entry) in travel_entries.iter().enumerate() {
|
||||||
|
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
|
||||||
|
let minutes = travel_data[ti].get(postcode.as_str()).map(|r| {
|
||||||
|
if entry.use_best {
|
||||||
|
r.best_minutes.unwrap_or(r.minutes)
|
||||||
|
} else {
|
||||||
|
r.minutes
|
||||||
|
}
|
||||||
|
});
|
||||||
|
match minutes {
|
||||||
|
Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
|
||||||
|
_ => return false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Aggregate for each postcode that has properties in bounds
|
// Aggregate for each postcode that has properties in bounds
|
||||||
// (polygon intersection check happens later when building response)
|
// (polygon intersection check happens later when building response)
|
||||||
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
|
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
|
||||||
|
// Travel time aggregation per postcode
|
||||||
|
let mut travel_aggs: FxHashMap<usize, Vec<TravelTimeAgg>> = FxHashMap::default();
|
||||||
|
|
||||||
for (&pc_idx, rows) in &postcode_rows {
|
for (&pc_idx, rows) in &postcode_rows {
|
||||||
let agg = postcode_aggs
|
let agg = postcode_aggs
|
||||||
.entry(pc_idx)
|
.entry(pc_idx)
|
||||||
|
|
@ -136,6 +197,24 @@ pub async fn get_postcodes(
|
||||||
agg.add_row(feature_data, row, num_features);
|
agg.add_row(feature_data, row, num_features);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Aggregate travel times for this postcode
|
||||||
|
if has_travel {
|
||||||
|
let postcode = &postcode_data.postcodes[pc_idx];
|
||||||
|
let tt_aggs = travel_aggs
|
||||||
|
.entry(pc_idx)
|
||||||
|
.or_insert_with(|| (0..travel_entries.len()).map(|_| TravelTimeAgg::new()).collect());
|
||||||
|
for (ti, entry) in travel_entries.iter().enumerate() {
|
||||||
|
if let Some(row_data) = travel_data[ti].get(postcode.as_str()) {
|
||||||
|
let minutes = if entry.use_best {
|
||||||
|
row_data.best_minutes.unwrap_or(row_data.minutes)
|
||||||
|
} else {
|
||||||
|
row_data.minutes
|
||||||
|
};
|
||||||
|
tt_aggs[ti].add(minutes as f32);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build response, filtering postcodes to only those whose polygon intersects query bounds
|
// Build response, filtering postcodes to only those whose polygon intersects query bounds
|
||||||
|
|
@ -218,6 +297,25 @@ pub async fn get_postcodes(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add travel time aggregation fields
|
||||||
|
if let Some(tt_aggs) = travel_aggs.get(&pc_idx) {
|
||||||
|
for (ti, agg) in tt_aggs.iter().enumerate() {
|
||||||
|
if agg.count > 0 {
|
||||||
|
let key = &travel_field_keys[ti];
|
||||||
|
let avg = agg.sum / agg.count as f64;
|
||||||
|
if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) {
|
||||||
|
props.insert(format!("min_{key}"), Value::Number(nm));
|
||||||
|
}
|
||||||
|
if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) {
|
||||||
|
props.insert(format!("max_{key}"), Value::Number(nm));
|
||||||
|
}
|
||||||
|
if let Some(nm) = serde_json::Number::from_f64(avg) {
|
||||||
|
props.insert(format!("avg_{key}"), Value::Number(nm));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Build GeoJSON Feature
|
// Build GeoJSON Feature
|
||||||
let mut feature = Map::new();
|
let mut feature = Map::new();
|
||||||
feature.insert("type".into(), Value::String("Feature".into()));
|
feature.insert("type".into(), Value::String("Feature".into()));
|
||||||
|
|
@ -241,6 +339,7 @@ pub async fn get_postcodes(
|
||||||
bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east),
|
bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east),
|
||||||
filters = num_filters,
|
filters = num_filters,
|
||||||
filters_raw = filters_str.as_deref().unwrap_or("-"),
|
filters_raw = filters_str.as_deref().unwrap_or("-"),
|
||||||
|
travel_entries = travel_entries.len(),
|
||||||
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
|
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
|
||||||
"GET /api/postcodes"
|
"GET /api/postcodes"
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -130,7 +130,7 @@ pub fn build_property(
|
||||||
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form",
|
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form",
|
||||||
),
|
),
|
||||||
duration: lookup_enum_value(
|
duration: lookup_enum_value(
|
||||||
feature_name_to_index, feature_data, num_features, enum_values, row, "Leashold/Freehold",
|
feature_name_to_index, feature_data, num_features, enum_values, row, "Leasehold/Freehold",
|
||||||
),
|
),
|
||||||
current_energy_rating: lookup_enum_value(
|
current_energy_rating: lookup_enum_value(
|
||||||
feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating",
|
feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating",
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,13 @@ pub async fn post_shorten(state: Arc<AppState>, Json(req): Json<ShortenRequest>)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_short_url(state: Arc<AppState>, Path(code): Path<String>) -> Response {
|
pub async fn get_short_url(state: Arc<AppState>, Path(code): Path<String>) -> Response {
|
||||||
|
if code.is_empty()
|
||||||
|
|| code.len() > 20
|
||||||
|
|| !code.bytes().all(|b| b.is_ascii_alphanumeric())
|
||||||
|
{
|
||||||
|
return StatusCode::BAD_REQUEST.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
let pb_url = state.pocketbase_url.trim_end_matches('/');
|
let pb_url = state.pocketbase_url.trim_end_matches('/');
|
||||||
|
|
||||||
let token = match auth_superuser(
|
let token = match auth_superuser(
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,62 @@
|
||||||
/// Per-hex-cell travel time aggregation.
|
/// A parsed travel time entry from the `travel` query parameter.
|
||||||
|
pub struct TravelEntry {
|
||||||
|
pub mode: String,
|
||||||
|
pub slug: String,
|
||||||
|
pub use_best: bool,
|
||||||
|
pub filter_min: Option<f32>,
|
||||||
|
pub filter_max: Option<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse `travel` param into a list of travel entries.
|
||||||
|
/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max`
|
||||||
|
pub fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
let mut seen_keys = Vec::new();
|
||||||
|
for segment in travel_str.split('|') {
|
||||||
|
let parts: Vec<&str> = segment.split(':').collect();
|
||||||
|
if parts.len() < 2 {
|
||||||
|
return Err(format!(
|
||||||
|
"each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'",
|
||||||
|
segment
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mode = parts[0].trim().to_string();
|
||||||
|
let slug = parts[1].trim().to_string();
|
||||||
|
|
||||||
|
let use_best = parts.len() >= 3 && parts[2].trim() == "best";
|
||||||
|
let filter_offset = if use_best { 1 } else { 0 };
|
||||||
|
|
||||||
|
let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset {
|
||||||
|
let min: f32 = parts[2 + filter_offset]
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
|
||||||
|
let max: f32 = parts[3 + filter_offset]
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
|
||||||
|
(Some(min), Some(max))
|
||||||
|
} else {
|
||||||
|
(None, None)
|
||||||
|
};
|
||||||
|
|
||||||
|
let key = format!("{}:{}", mode, slug);
|
||||||
|
if seen_keys.contains(&key) {
|
||||||
|
return Err(format!("duplicate travel entry '{}'", key));
|
||||||
|
}
|
||||||
|
seen_keys.push(key);
|
||||||
|
entries.push(TravelEntry {
|
||||||
|
mode,
|
||||||
|
slug,
|
||||||
|
use_best,
|
||||||
|
filter_min,
|
||||||
|
filter_max,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-cell travel time aggregation.
|
||||||
pub struct TravelTimeAgg {
|
pub struct TravelTimeAgg {
|
||||||
pub min: f32,
|
pub min: f32,
|
||||||
pub max: f32,
|
pub max: f32,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue