From 9179acd4cdd11e8f4e2c749d6c89d581e708a0b3 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 1 Feb 2026 21:00:59 +0000 Subject: [PATCH 01/66] Optimisations --- README.md | 65 +++-- frontend/src/App.tsx | 36 ++- frontend/src/components/AreaPane.tsx | 334 ++++++++++++++++++++++++-- frontend/src/components/Map.tsx | 13 + frontend/src/index.tsx | 3 + frontend/src/types.ts | 1 + frontend/src/usePlausible.ts | 73 ++++++ frontend/webpack.config.js | 6 + server-rs/Cargo.lock | 10 + server-rs/Cargo.toml | 1 + server-rs/src/consts.rs | 5 +- server-rs/src/features.rs | 4 +- server-rs/src/filter.rs | 10 +- server-rs/src/grid_index.rs | 29 ++- server-rs/src/main.rs | 13 +- server-rs/src/routes/features.rs | 6 +- server-rs/src/routes/hexagon_stats.rs | 67 ++++-- server-rs/src/routes/hexagons.rs | 30 +-- server-rs/src/routes/pois.rs | 8 +- server-rs/src/routes/properties.rs | 48 ++-- server-rs/src/tests.rs | 30 +-- 21 files changed, 653 insertions(+), 139 deletions(-) create mode 100644 frontend/src/usePlausible.ts diff --git a/README.md b/README.md index 3c19dc5..e63ab17 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ ```sh curl -1sLf 'https://dl.cloudsmith.io/public/task/task/setup.deb.sh' | sudo -E bash +apt install task task prepare ``` @@ -34,30 +35,14 @@ task prepare 5. fibre optic availability 6. between london and bournemouth ish - 7. [Y] historical prices 8. current listings -## Action plan - -1. use openstreetmap api to get the map - -## Data Sources - -- [Price Paid](https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads) - [English Indices of Deprevation 2025](https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025) - The English Indices of Deprivation (IoD25) measure relative levels of deprivation in 33,755 small areas or neighbourhoods, called Lower-layer Super Output Areas (LSOAs), in England. -- [Population by Ethnicity and Region 2021](https://www.ethnicity-facts-figures.service.gov.uk/uk-population-by-ethnicity/national-and-regional-populations/regional-ethnic-diversity/latest/#download-the-data) - -- [Crime](https://data.police.uk/data/) - -- [Postcode -> GPS](https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data) - -Nice to haves? - - - file 8! ## Backend Data Sources @@ -70,10 +55,54 @@ Nice to haves? - [Open Geography](https://geoportal.statistics.gov.uk/) - [CommunitiesOpenData](https://communitiesopendata-communities.hub.arcgis.com/) - [PlanetOSM](https://planet.openstreetmap.org/) for open street map POI -- [TFL api](https://api-portal.tfl.gov.uk/signin) -- [EPC](https://epc.opendatacommunities.org/login) - +- [naptan](https://naptan.dft.gov.uk/naptan/schema/2.4/doc/NaPTANSchemaGuide-2.4-v0.57.pdf) + rightmove: curl '' curl '' + + + +Make it mobile friendly + +Serve the frontend from the server + +Add an account management page + + + +categories the categories + + +mkdir -p data/crime +unzip data/d29f0314840ef7dcbb5cde66e383fe08059dab5a.zip -d data/crime/ +rm data/d29f0314840ef7dcbb5cde66e383fe08059dab5a.zip + + +https://xploria.co.uk/data-sources/ + + +panning is slow + + + +epc oopt out https://www.gov.uk/guidance/energy-performance-certificates-opt-out-of-public-disclosure + +righmove lins + +❯ Use the approach you described. Look at the current state of the frontend and backend and add user management to it. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be + prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. There must be a use page showing the lifetime + license key. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring + of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too. Add a docker compose file to host pocketbase and the rust server. While doing this, protect the + server against DOS-ing. Once you're logged in, you can save your searches and then look at your saved searches. Instead of just a share button in the header, add a save search one for logged in users and a login button (on all pages) for not-logged in users. Don't + support profile pictures or full names to avoid GDPR requirements. + + + + embedd google street view + + + + how to handle too many pois \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0de2505..84af9b7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,5 @@ import { useState, useEffect, useCallback, useRef, useMemo } from 'react'; +import { trackPageview } from './usePlausible'; import Map from './components/Map'; import Filters from './components/Filters'; import POIPane from './components/POIPane'; @@ -57,6 +58,11 @@ async function fetchWithRetry( // Detect if running through VS Code web proxy and construct API base URL function getApiBaseUrl(): string { + // In production builds, always use same-origin (Rust server serves both API and frontend) + if (process.env.NODE_ENV === 'production') { + return ''; + } + const { pathname, href } = window.location; // Check pathname for /proxy/PORT pattern (VS Code web proxy) @@ -71,7 +77,7 @@ function getApiBaseUrl(): string { return `${hrefMatch[1]}8001`; } - // Default: same origin (works for both local dev with webpack proxy and production) + // Default: same origin (works for local dev with webpack proxy) return ''; } @@ -417,6 +423,7 @@ export default function App() { const url = hash ? `${window.location.pathname}${window.location.search}#${hash}` : `${window.location.pathname}${window.location.search}`; window.history.pushState({ page }, '', url); setActivePage(page); + trackPageview(); }, []); // Handle browser back/forward @@ -465,8 +472,9 @@ export default function App() { // Derive view feature: active drag takes priority over pinned const viewFeature = activeFeature || pinnedFeature; const viewSource: 'drag' | 'eye' | null = activeFeature ? 'drag' : pinnedFeature ? 'eye' : null; - // Color range: always the feature's full slider range from metadata - // For enum features, use ordinal index range [0, values.length - 1] + // Color range: use the filter slider range when a numeric filter is active, + // otherwise fall back to the feature's full range from metadata. + // For enum features, use ordinal index range [0, values.length - 1]. const colorRange = useMemo((): [number, number] | null => { if (!viewFeature) return null; const meta = features.find((f) => f.name === viewFeature); @@ -474,9 +482,13 @@ export default function App() { if (meta.type === 'enum' && meta.values && meta.values.length > 0) { return [0, meta.values.length - 1]; } + // Use live drag values or committed filter range if available + if (activeFeature === viewFeature && dragValue) return dragValue; + const filterVal = filters[viewFeature]; + if (filterVal && typeof filterVal[0] === 'number') return filterVal as [number, number]; if (meta.min != null && meta.max != null) return [meta.min, meta.max]; return null; - }, [viewFeature, features]); + }, [viewFeature, features, activeFeature, dragValue, filters]); // Filter range: current drag or committed filter values, used for gray-out const filterRange = useMemo((): [number, number] | null => { @@ -1087,6 +1099,22 @@ export default function App() { onHoverModeChange={setHoverMode} onViewProperties={handleViewPropertiesFromArea} onClose={handleCloseProperties} + hexagonLocation={ + (() => { + const hexId = hoverMode && hoveredHexagon && hoveredHexagon !== selectedHexagon?.h3 + ? hoveredHexagon + : selectedHexagon?.h3; + const hex = hexId ? data.find((d) => d.h3 === hexId) : null; + if (!hex || typeof hex.lat !== 'number' || typeof hex.lon !== 'number') return null; + return { + lat: hex.lat as number, + lon: hex.lon as number, + postcode: (hex.postcode as string | undefined) ?? null, + resolution, + }; + })() + } + filters={filters} /> ) : rightPaneTab === 'properties' ? ( void; onViewProperties: () => void; onClose: () => void; + hexagonLocation: HexagonLocation | null; + filters: FeatureFilters; } function formatValue(value: number): string { @@ -37,10 +46,7 @@ function groupFeatures( return groups; } -function MiniHistogram({ counts, maxCount }: { counts: number[]; maxCount: number }) { - if (maxCount === 0) return null; - // Downsample to ~20 bars for display - const targetBars = 20; +function downsampleBars(counts: number[], targetBars: number): number[] { const step = Math.max(1, Math.floor(counts.length / targetBars)); const bars: number[] = []; for (let index = 0; index < counts.length; index += step) { @@ -50,21 +56,268 @@ function MiniHistogram({ counts, maxCount }: { counts: number[]; maxCount: numbe } bars.push(sum); } - const barMax = Math.max(...bars, 1); + return bars; +} + +function DualHistogram({ + localCounts, + globalCounts, + min, + max, + globalMean, +}: { + localCounts: number[]; + globalCounts: number[]; + min: number; + max: number; + globalMean?: number; +}) { + const targetBars = 25; + const localBars = downsampleBars(localCounts, targetBars); + const globalBars = downsampleBars(globalCounts, targetBars); + + const barCount = Math.min(localBars.length, globalBars.length); + const localMax = Math.max(...localBars, 1); + const globalMax = Math.max(...globalBars, 1); + + const meanFraction = + globalMean != null && max > min ? (globalMean - min) / (max - min) : null; return ( -
- {bars.map((count, index) => ( -
0 ? 1 : 0.1 }} - /> +
+
+ {Array.from({ length: barCount }).map((_, index) => { + const globalHeight = (globalBars[index] / globalMax) * 100; + const localHeight = (localBars[index] / localMax) * 100; + return ( +
+
+
0 ? 1 : 0.1, + }} + /> +
+ ); + })} + {meanFraction != null && meanFraction >= 0 && meanFraction <= 1 && ( +
+ )} +
+
+ ); +} + +function SkeletonHistogram() { + return ( +
+
+
+
+
+
+ {Array.from({ length: 15 }).map((_, i) => ( +
+ ))} +
+
+
+
+
+
+ ); +} + +function LoadingSkeleton() { + return ( +
+ {[0, 1, 2].map((groupIdx) => ( +
+
+
+ {Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => ( + + ))} +
+
))}
); } +// Map app property types to each site's expected values +const PROPERTY_TYPE_MAP: Record = { + 'House': { rightmove: 'detached,semi-detached,terraced', onthemarket: 'property', zoopla: '' }, + 'Detached': { rightmove: 'detached', onthemarket: 'detached', zoopla: 'detached' }, + 'Semi-Detached': { rightmove: 'semi-detached', onthemarket: 'semi-detached', zoopla: 'semi_detached' }, + 'Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, + 'End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, + 'Enclosed Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, + 'Enclosed End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, + 'Flat': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' }, + 'Maisonette': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' }, + 'Bungalow': { rightmove: 'bungalow', onthemarket: 'bungalow', zoopla: 'bungalow' }, + 'Park home': { rightmove: 'park-home', onthemarket: 'property', zoopla: '' }, +}; + +// Approximate H3 hex edge length in miles by resolution +// See https://h3geo.org/docs/core-library/restable +const H3_RADIUS_MILES: Record = { + 4: 15, // ~24km edge → ~15mi + 5: 6, // ~9km → ~6mi + 6: 3, // ~3.5km → ~3mi + 7: 1, // ~1.3km → ~1mi + 8: 0.5, // ~0.5km → ~0.3mi, round up + 9: 0.25, // ~0.17km + 10: 0.25, // ~0.07km + 11: 0.25, // ~0.025km + 12: 0.25, +}; + +// Rightmove only accepts specific radius values +const RIGHTMOVE_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40]; +// OnTheMarket and Zoopla accept similar sets +const OTM_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40]; +const ZOOPLA_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 25, 30]; + +function nearestRadius(target: number, allowed: number[]): number { + return allowed.reduce((best, r) => (Math.abs(r - target) < Math.abs(best - target) ? r : best)); +} + +function buildPropertySearchUrls( + location: HexagonLocation, + filters: FeatureFilters +): { rightmove: string; onthemarket: string; zoopla: string } { + const { lat, lon, postcode, resolution } = location; + const radiusMiles = H3_RADIUS_MILES[resolution] ?? 1; + const coordStr = `${lat.toFixed(5)},${lon.toFixed(5)}`; + + // Extract price filters + const priceFilter = filters['Last known price']; + const minPrice = Array.isArray(priceFilter) && typeof priceFilter[0] === 'number' ? priceFilter[0] : undefined; + const maxPrice = Array.isArray(priceFilter) && typeof priceFilter[1] === 'number' ? priceFilter[1] : undefined; + + // Extract property type filters + const propertyTypes = filters['Property type']; + const selectedTypes = Array.isArray(propertyTypes) && typeof propertyTypes[0] === 'string' ? propertyTypes as string[] : []; + + // --- Rightmove --- + // Rightmove accepts both postcodes and lat,lon in searchLocation + const rmParams = new URLSearchParams(); + rmParams.set('searchLocation', postcode || coordStr); + rmParams.set('channel', 'BUY'); + rmParams.set('radius', String(nearestRadius(radiusMiles, RIGHTMOVE_RADII))); + if (minPrice !== undefined) rmParams.set('minPrice', String(Math.round(minPrice))); + if (maxPrice !== undefined) rmParams.set('maxPrice', String(Math.round(maxPrice))); + if (selectedTypes.length > 0) { + const rmTypes = [...new Set(selectedTypes.flatMap((t) => { + const mapped = PROPERTY_TYPE_MAP[t]?.rightmove; + return mapped ? mapped.split(',') : []; + }))]; + if (rmTypes.length > 0) rmParams.set('propertyTypes', rmTypes.join(',')); + } + const rightmove = `https://www.rightmove.co.uk/property-for-sale/find.html?${rmParams.toString()}`; + + // --- OnTheMarket --- + let otmType = 'property'; + if (selectedTypes.length > 0) { + const otmTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.onthemarket).filter(Boolean))]; + if (otmTypes.length === 1 && otmTypes[0] !== 'property') otmType = otmTypes[0]!; + } + const otmParams = new URLSearchParams(); + otmParams.set('radius', String(nearestRadius(radiusMiles, OTM_RADII))); + if (minPrice !== undefined) otmParams.set('min-price', String(Math.round(minPrice))); + if (maxPrice !== undefined) otmParams.set('max-price', String(Math.round(maxPrice))); + let onthemarket: string; + if (postcode) { + const slug = postcode.replace(/\s+/g, '-').toLowerCase(); + onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/${slug}/?${otmParams.toString()}`; + } else { + // Use lat/lon search with geo params for bigger hexagons without a postcode + otmParams.set('search-site', 'geo'); + otmParams.set('geo-lat', String(lat)); + otmParams.set('geo-lng', String(lon)); + onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/?${otmParams.toString()}`; + } + + // --- Zoopla --- + const zParams = new URLSearchParams(); + zParams.set('q', postcode || coordStr); + zParams.set('search_source', 'for-sale'); + zParams.set('radius', String(nearestRadius(radiusMiles, ZOOPLA_RADII))); + if (minPrice !== undefined) zParams.set('price_min', String(Math.round(minPrice))); + if (maxPrice !== undefined) zParams.set('price_max', String(Math.round(maxPrice))); + if (selectedTypes.length > 0) { + const zTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.zoopla).filter(Boolean))]; + for (const zt of zTypes) { + zParams.append('property_sub_type', zt!); + } + } + let zoopla: string; + if (postcode) { + const slug = postcode.replace(/\s+/g, '-').toLowerCase(); + zoopla = `https://www.zoopla.co.uk/for-sale/property/${slug}/?${zParams.toString()}`; + } else { + // Use coordinate-based path for bigger hexagons + zParams.set('geo_autocomplete_identifier', `geo_${lat}_${lon}`); + zoopla = `https://www.zoopla.co.uk/for-sale/property/?${zParams.toString()}`; + } + + return { rightmove, onthemarket, zoopla }; +} + +function ExternalSearchLinks({ location, filters }: { location: HexagonLocation; filters: FeatureFilters }) { + const urls = useMemo(() => buildPropertySearchUrls(location, filters), [location, filters]); + const radiusMiles = H3_RADIUS_MILES[location.resolution] ?? 1; + const label = location.postcode || `${radiusMiles}mi radius`; + + return ( +
+

+ Search {label} on +

+ +
+ ); +} + function EnumBarChart({ counts }: { counts: Record }) { const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA); const maxCount = Math.max(...entries.map(([, count]) => count), 1); @@ -99,6 +352,8 @@ export default function AreaPane({ onHoverModeChange, onViewProperties, onClose, + hexagonLocation, + filters, }: AreaPaneProps) { const featureGroups = useMemo(() => groupFeatures(globalFeatures), [globalFeatures]); @@ -113,6 +368,12 @@ export default function AreaPane({ return new Map(stats.enum_features.map((feature) => [feature.name, feature])); }, [stats]); + // Build lookup for global feature metadata (for histogram overlay) + const globalFeatureByName = useMemo( + () => new Map(globalFeatures.map((f) => [f.name, f])), + [globalFeatures] + ); + if (!hexagonId) { return (
@@ -174,10 +435,13 @@ export default function AreaPane({ )}
+ {/* External search links */} + {hexagonLocation && stats && } + {/* Stats content */}
{loading && !stats ? ( -
Loading...
+ ) : stats ? (
{featureGroups.map((group) => { @@ -198,9 +462,24 @@ export default function AreaPane({ const enumStats = enumByName.get(feature.name); if (numericStats) { - const maxCount = Math.max(...numericStats.histogram.counts); + const globalFeature = globalFeatureByName.get(feature.name); + const globalHistogram = globalFeature?.histogram; + // Compute a global mean from the global histogram for the mean line + let globalMean: number | undefined; + if (globalHistogram && globalHistogram.counts.length > 0) { + const totalCount = globalHistogram.counts.reduce((a, b) => a + b, 0); + if (totalCount > 0) { + let weightedSum = 0; + for (let i = 0; i < globalHistogram.counts.length; i++) { + const binCenter = globalHistogram.min + (i + 0.5) * globalHistogram.bin_width; + weightedSum += binCenter * globalHistogram.counts[i]; + } + globalMean = weightedSum / totalCount; + } + } + return ( -
+
{feature.name} @@ -210,17 +489,32 @@ export default function AreaPane({
- {formatValue(numericStats.min)} - {formatValue(numericStats.max)} + {formatValue(numericStats.histogram.min)} + {formatValue(numericStats.histogram.max)}
- + {globalHistogram ? ( + + ) : ( + + )}
); } if (enumStats) { return ( -
+
{feature.name} diff --git a/frontend/src/components/Map.tsx b/frontend/src/components/Map.tsx index 8091e80..82c33b8 100644 --- a/frontend/src/components/Map.tsx +++ b/frontend/src/components/Map.tsx @@ -654,6 +654,19 @@ export default memo(function Map({ + {viewSource === 'eye' && viewFeature && ( +
+ + Previewing “{viewFeature}” + + +
+ )} {viewFeature && colorRange && colorFeatureMeta ? ( ; + revenue?: { currency: string; amount: number }; +}; + +function sendEvent(name: string, options?: EventOptions) { + const payload: Record = { + n: name, + u: window.location.href, + d: DOMAIN, + r: document.referrer || null, + }; + if (options?.props) { + payload.p = JSON.stringify(options.props); + } + if (options?.revenue) { + payload.$ = JSON.stringify(options.revenue); + } + if (navigator.sendBeacon) { + navigator.sendBeacon( + ENDPOINT, + new Blob([JSON.stringify(payload)], { type: 'application/json' }) + ); + } else { + fetch(ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + keepalive: true, + }).catch(() => {}); + } +} + +let initialized = false; + +/** + * Tracks pageview on first call and returns a trackEvent function. + * Tracks outbound link clicks automatically. + */ +export function initPlausible() { + if (initialized) return; + initialized = true; + + // Initial pageview + sendEvent('pageview'); + + // Track outbound link clicks + document.addEventListener('click', (e) => { + const link = (e.target as HTMLElement).closest?.('a'); + if (!link) return; + const href = link.getAttribute('href'); + if (!href) return; + try { + const url = new URL(href, window.location.origin); + if (url.hostname !== window.location.hostname) { + sendEvent('Outbound Link: Click', { props: { url: href } }); + } + } catch { + // invalid URL, ignore + } + }); +} + +export function trackPageview(options?: EventOptions) { + sendEvent('pageview', options); +} + +export function trackEvent(name: string, options?: EventOptions) { + sendEvent(name, options); +} diff --git a/frontend/webpack.config.js b/frontend/webpack.config.js index 15b9889..f0c4082 100644 --- a/frontend/webpack.config.js +++ b/frontend/webpack.config.js @@ -51,6 +51,12 @@ module.exports = (env, argv) => { context: ['/api'], target: 'http://localhost:8001', }, + { + context: ['/status'], + target: 'https://stats.schmelczer.dev', + changeOrigin: true, + pathRewrite: { '^/status': '/api/event' }, + }, ], }, }; diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 89c3585..68fd1a9 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -1031,6 +1031,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1813,6 +1822,7 @@ dependencies = [ "axum", "clap", "h3o", + "lasso", "polars", "rayon", "rustc-hash", diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index fd0d22b..5df1660 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -14,6 +14,7 @@ h3o = "0.7" serde = { version = "1", features = ["derive"] } serde_json = "1" rayon = "1" +lasso = "0.7" rustc-hash = "2" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } diff --git a/server-rs/src/consts.rs b/server-rs/src/consts.rs index 2b57f86..f376e1a 100644 --- a/server-rs/src/consts.rs +++ b/server-rs/src/consts.rs @@ -1,12 +1,15 @@ pub const HISTOGRAM_BINS: usize = 100; -pub const H3_PRECOMPUTE_MIN: u8 = 4; +pub const H3_PRECOMPUTE_MIN: u8 = 7; pub const H3_PRECOMPUTE_MAX: u8 = 12; +pub const H3_REQUEST_MIN: u8 = 4; +pub const H3_REQUEST_MAX: u8 = 12; pub const SERVER_ADDRESS: &str = "0.0.0.0:8001"; pub const BOUNDS_QUANTIZATION: f64 = 0.01; pub const BOUNDS_BUFFER_PERCENT: f64 = 0.1; +pub const GRID_CELL_SIZE: f32 = 0.01; pub const POSTCODE_MIN_RESOLUTION: u8 = 11; pub const MAX_POIS_PER_REQUEST: usize = 2500; pub const DEFAULT_PROPERTIES_LIMIT: usize = 100; diff --git a/server-rs/src/features.rs b/server-rs/src/features.rs index 9af4f27..ddc58ba 100644 --- a/server-rs/src/features.rs +++ b/server-rs/src/features.rs @@ -3,7 +3,7 @@ pub enum Bounds { /// Fixed min/max values for the slider - Fixed { min: f64, max: f64 }, + Fixed { min: f32, max: f32 }, /// Compute percentile from data at startup Percentile { low: f64, high: f64 }, } @@ -13,7 +13,7 @@ pub struct FeatureConfig { pub name: &'static str, pub bounds: Bounds, /// Slider step size. Controls the granularity of the range slider in the UI. - pub step: f64, + pub step: f32, /// Short one-line description shown in the filter sidebar pub description: &'static str, /// Longer description explaining methodology, data source, and caveats diff --git a/server-rs/src/filter.rs b/server-rs/src/filter.rs index 7be3d24..69d6e63 100644 --- a/server-rs/src/filter.rs +++ b/server-rs/src/filter.rs @@ -3,8 +3,8 @@ use crate::data::EnumFeatureData; pub struct ParsedFilter { pub feat_idx: usize, - pub min: f64, - pub max: f64, + pub min: f32, + pub max: f32, } pub struct ParsedEnumFilter { @@ -51,11 +51,11 @@ pub fn parse_filters( if num_parts.len() != 2 { continue; } - let min = match num_parts[0].trim().parse::() { + let min = match num_parts[0].trim().parse::() { Ok(value) => value, Err(_) => continue, }; - let max = match num_parts[1].trim().parse::() { + let max = match num_parts[1].trim().parse::() { Ok(value) => value, Err(_) => continue, }; @@ -72,7 +72,7 @@ pub fn row_passes_filters( row: usize, filters: &[ParsedFilter], enum_filters: &[ParsedEnumFilter], - feature_data: &[f64], + feature_data: &[f32], num_features: usize, enum_features: &[EnumFeatureData], ) -> bool { diff --git a/server-rs/src/grid_index.rs b/server-rs/src/grid_index.rs index f5b57c7..6849cbe 100644 --- a/server-rs/src/grid_index.rs +++ b/server-rs/src/grid_index.rs @@ -3,9 +3,9 @@ /// Divides the UK bounding box into cells of ~0.01 degrees (~1km), /// each storing indices of rows whose lat/lon falls within that cell. pub struct GridIndex { - min_lat: f64, - min_lon: f64, - cell_size: f64, + min_lat: f32, + min_lon: f32, + cell_size: f32, cols: usize, rows: usize, /// cells[row * cols + col] = vec of row indices @@ -13,11 +13,11 @@ pub struct GridIndex { } impl GridIndex { - pub fn build(lat: &[f64], lon: &[f64], cell_size: f64) -> Self { - let mut min_lat = f64::INFINITY; - let mut max_lat = f64::NEG_INFINITY; - let mut min_lon = f64::INFINITY; - let mut max_lon = f64::NEG_INFINITY; + pub fn build(lat: &[f32], lon: &[f32], cell_size: f32) -> Self { + let mut min_lat = f32::INFINITY; + let mut max_lat = f32::NEG_INFINITY; + let mut min_lon = f32::INFINITY; + let mut max_lon = f32::NEG_INFINITY; for index in 0..lat.len() { if lat[index] < min_lat { @@ -71,6 +71,7 @@ impl GridIndex { } } + /// Query accepts f64 bounds (from HTTP parsing) and casts internally. pub fn query(&self, south: f64, west: f64, north: f64, east: f64) -> Vec { let Some((row_min, row_max, col_min, col_max)) = self.clamp_bounds(south, west, north, east) @@ -121,10 +122,14 @@ impl GridIndex { north: f64, east: f64, ) -> Option<(usize, usize, usize, usize)> { - let row_min_raw = ((south - self.min_lat) / self.cell_size) as isize; - let row_max_raw = ((north - self.min_lat) / self.cell_size) as isize; - let col_min_raw = ((west - self.min_lon) / self.cell_size) as isize; - let col_max_raw = ((east - self.min_lon) / self.cell_size) as isize; + let min_lat = self.min_lat as f64; + let min_lon = self.min_lon as f64; + let cell_size = self.cell_size as f64; + + let row_min_raw = ((south - min_lat) / cell_size) as isize; + let row_max_raw = ((north - min_lat) / cell_size) as isize; + let col_min_raw = ((west - min_lon) / cell_size) as isize; + let col_max_raw = ((east - min_lon) / cell_size) as isize; let row_min = row_min_raw.max(0) as usize; let row_max_clamped = row_max_raw.min(self.rows as isize - 1); diff --git a/server-rs/src/main.rs b/server-rs/src/main.rs index 71b18ea..eb5d569 100644 --- a/server-rs/src/main.rs +++ b/server-rs/src/main.rs @@ -69,7 +69,7 @@ async fn main() -> anyhow::Result<()> { ); info!("Building spatial grid index (0.01° cells)"); - let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, 0.01); + let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, consts::GRID_CELL_SIZE); info!( "Precomputing H3 cells for resolutions {}-{}", @@ -89,7 +89,7 @@ async fn main() -> anyhow::Result<()> { info!(pois = poi_data.lat.len(), "POI data loaded"); info!("Building POI spatial grid index"); - let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, 0.01); + let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE); let min_keys: Vec = property_data .feature_names @@ -116,11 +116,14 @@ async fn main() -> anyhow::Result<()> { let poi_category_groups = { let mut group_cats: std::collections::HashMap> = std::collections::HashMap::new(); - for (category, group) in poi_data.category.iter().zip(poi_data.group.iter()) { + let num_pois = poi_data.category.indices.len(); + for row in 0..num_pois { + let category = poi_data.category.get(row).to_string(); + let group = poi_data.group.get(row).to_string(); group_cats - .entry(group.clone()) + .entry(group) .or_default() - .insert(category.clone()); + .insert(category); } // Validate that data groups match the hardcoded order exactly let expected: std::collections::HashSet<&str> = diff --git a/server-rs/src/routes/features.rs b/server-rs/src/routes/features.rs index e7081ad..45611cc 100644 --- a/server-rs/src/routes/features.rs +++ b/server-rs/src/routes/features.rs @@ -14,9 +14,9 @@ pub enum FeatureInfo { #[serde(rename = "numeric")] Numeric { name: String, - min: f64, - max: f64, - step: f64, + min: f32, + max: f32, + step: f32, histogram: Histogram, description: &'static str, detail: &'static str, diff --git a/server-rs/src/routes/hexagon_stats.rs b/server-rs/src/routes/hexagon_stats.rs index 6f3d357..0f5182c 100644 --- a/server-rs/src/routes/hexagon_stats.rs +++ b/server-rs/src/routes/hexagon_stats.rs @@ -8,7 +8,7 @@ use axum::response::IntoResponse; use serde::Deserialize; use tracing::{info, warn}; -use crate::consts::{ENUM_NULL, HISTOGRAM_BINS}; +use crate::consts::{ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, HISTOGRAM_BINS}; use crate::filter::{parse_filters, row_passes_filters}; use crate::state::AppState; @@ -31,17 +31,21 @@ pub async fn get_hexagon_stats( })?; let cell_u64: u64 = cell.into(); - let resolution = params.resolution as usize; - if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() { + let resolution = params.resolution; + if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) { warn!( resolution, - "Invalid or non-precomputed resolution for hexagon-stats" + "Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX ); return Err(( StatusCode::BAD_REQUEST, - "Invalid or non-precomputed resolution".to_string(), + format!( + "resolution must be between {} and {}", + H3_REQUEST_MIN, H3_REQUEST_MAX + ), )); } + let resolution_idx = resolution as usize; let h3_str = params.h3.clone(); let filters_str = params.filters.clone(); @@ -54,7 +58,13 @@ pub async fn get_hexagon_stats( let result = tokio::task::spawn_blocking(move || { let start_time = std::time::Instant::now(); - let h3_data = &state.h3_cells[resolution]; + let precomputed: Option<&[u64]> = state + .h3_cells + .get(resolution_idx) + .filter(|cells| !cells.is_empty()) + .map(|cells| cells.as_slice()); + let h3_res = h3o::Resolution::try_from(resolution) + .map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?; let num_features = state.data.num_features; let feature_data = &state.data.feature_data; let enum_features = &state.data.enum_features; @@ -67,7 +77,14 @@ pub async fn get_hexagon_stats( .grid .for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| { let row = row_idx as usize; - if h3_data[row] == cell_u64 + let row_cell = if let Some(h3_data) = precomputed { + h3_data[row] + } else { + h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) + .map(|coord| u64::from(coord.to_cell(h3_res))) + .unwrap_or(0) + }; + if row_cell == cell_u64 && row_passes_filters( row, &parsed_filters, @@ -98,9 +115,9 @@ pub async fn get_hexagon_stats( let bin_width = global_stats.histogram.bin_width; let mut count = 0usize; - let mut min_value = f64::INFINITY; - let mut max_value = f64::NEG_INFINITY; - let mut sum = 0.0f64; + let mut min_value = f32::INFINITY; + let mut max_value = f32::NEG_INFINITY; + let mut sum = 0.0f64; // keep f64 for mean precision let mut bins = vec![0u64; HISTOGRAM_BINS]; for &row in &matching_rows { @@ -113,12 +130,12 @@ pub async fn get_hexagon_stats( if value > max_value { max_value = value; } - sum += value; + sum += value as f64; - // Bin into histogram using global edges + // Bin into histogram using global edges (cast to f64 for bin index math) if bin_width > 0.0 { let bin_index = - ((value - histogram_min) / bin_width).floor() as isize; + ((value as f64 - histogram_min as f64) / bin_width as f64).floor() as isize; let clamped_index = bin_index.max(0).min((HISTOGRAM_BINS - 1) as isize) as usize; bins[clamped_index] += 1; } @@ -138,15 +155,15 @@ pub async fn get_hexagon_stats( output.push_str("{\"name\":"); write_json_string(&mut output, feature_name); write!(output, ",\"count\":{}", count).unwrap(); - write!(output, ",\"min\":{}", format_f64(min_value)).unwrap(); - write!(output, ",\"max\":{}", format_f64(max_value)).unwrap(); + write!(output, ",\"min\":{}", format_num(min_value)).unwrap(); + write!(output, ",\"max\":{}", format_num(max_value)).unwrap(); write!(output, ",\"mean\":{}", format_f64(mean)).unwrap(); output.push_str(",\"histogram\":{\"min\":"); - write!(output, "{}", format_f64(histogram_min)).unwrap(); + write!(output, "{}", format_num(histogram_min)).unwrap(); output.push_str(",\"max\":"); - write!(output, "{}", format_f64(histogram_max)).unwrap(); + write!(output, "{}", format_num(histogram_max)).unwrap(); output.push_str(",\"bin_width\":"); - write!(output, "{}", format_f64(bin_width)).unwrap(); + write!(output, "{}", format_num(bin_width)).unwrap(); output.push_str(",\"counts\":["); for (bin_index, &bin_count) in bins.iter().enumerate() { if bin_index > 0 { @@ -216,10 +233,11 @@ pub async fn get_hexagon_stats( "GET /api/hexagon-stats" ); - output + Ok(output) }) .await - .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?; + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))? + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; Ok(( [(axum::http::header::CONTENT_TYPE, "application/json")], @@ -242,6 +260,15 @@ fn write_json_string(output: &mut String, value: &str) { output.push('"'); } +fn format_num(value: f32) -> String { + let fv = value as f64; + if fv.fract() == 0.0 && fv.abs() < 1e15 { + format!("{:.1}", fv) + } else { + format!("{}", fv) + } +} + fn format_f64(value: f64) -> String { if value.fract() == 0.0 && value.abs() < 1e15 { format!("{:.1}", value) diff --git a/server-rs/src/routes/hexagons.rs b/server-rs/src/routes/hexagons.rs index 27ed200..a515825 100644 --- a/server-rs/src/routes/hexagons.rs +++ b/server-rs/src/routes/hexagons.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use tracing::{info, warn}; use crate::consts::{ - BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_PRECOMPUTE_MIN, + BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, POSTCODE_MIN_RESOLUTION, }; use crate::filter::parse_filters; @@ -44,8 +44,8 @@ pub struct HexagonParams { /// Per-cell accumulator for aggregating features struct CellAgg { count: u32, - mins: Vec, - maxs: Vec, + mins: Vec, + maxs: Vec, /// Min/max ordinal indices for enum features (255 = no data yet) enum_mins: Vec, enum_maxs: Vec, @@ -60,8 +60,8 @@ impl CellAgg { fn new(num_features: usize, num_enums: usize) -> Self { CellAgg { count: 0, - mins: vec![f64::INFINITY; num_features], - maxs: vec![f64::NEG_INFINITY; num_features], + mins: vec![f32::INFINITY; num_features], + maxs: vec![f32::NEG_INFINITY; num_features], enum_mins: vec![ENUM_NULL; num_enums], enum_maxs: vec![0; num_enums], postcode: None, @@ -75,7 +75,7 @@ impl CellAgg { /// feature_data[row * num_features + feat_idx] — all features for one row /// are contiguous, so this reads a single cache line per ~8 features. #[inline] - fn add_row(&mut self, feature_data: &[f64], row: usize, num_features: usize) { + fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) { self.count += 1; let base = row * num_features; let row_slice = &feature_data[base..base + num_features]; @@ -110,9 +110,9 @@ impl CellAgg { /// Track postcode and centroid for high-resolution cells. /// Uses simple "first seen" approach — at res 11/12, most rows in a cell share a postcode. #[inline] - fn add_postcode(&mut self, postcode: &str, lat: f64, lon: f64) { - self.lat_sum += lat; - self.lon_sum += lon; + fn add_postcode(&mut self, postcode: &str, lat: f32, lon: f32) { + self.lat_sum += lat as f64; + self.lon_sum += lon as f64; if postcode.is_empty() { return; } @@ -212,16 +212,16 @@ pub async fn get_hexagons( Query(params): Query, ) -> Result { let resolution = params.resolution; - if resolution < H3_PRECOMPUTE_MIN || resolution > H3_PRECOMPUTE_MAX { + if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) { warn!( resolution, - "Resolution out of range [{}, {}]", H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX + "Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX ); return Err(( StatusCode::BAD_REQUEST, format!( "resolution must be between {} and {}", - H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX + H3_REQUEST_MIN, H3_REQUEST_MAX ), )); } @@ -304,7 +304,7 @@ pub async fn get_hexagons( aggregation.add_enums(enum_features, row); if include_postcode { aggregation.add_postcode( - &state.data.postcode[row], + state.data.postcode(row), state.data.lat[row], state.data.lon[row], ); @@ -320,7 +320,7 @@ pub async fn get_hexagons( if !row_passes(row) { return; } - let cell_id = h3o::LatLng::new(state.data.lat[row], state.data.lon[row]) + let cell_id = h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) .map(|coord| u64::from(coord.to_cell(h3_res))) .unwrap_or(0); let aggregation = groups @@ -330,7 +330,7 @@ pub async fn get_hexagons( aggregation.add_enums(enum_features, row); if include_postcode { aggregation.add_postcode( - &state.data.postcode[row], + state.data.postcode(row), state.data.lat[row], state.data.lon[row], ); diff --git a/server-rs/src/routes/pois.rs b/server-rs/src/routes/pois.rs index f00008f..f4569f3 100644 --- a/server-rs/src/routes/pois.rs +++ b/server-rs/src/routes/pois.rs @@ -55,7 +55,7 @@ pub async fn get_pois( .filter_map(|&row_idx| { let row = row_idx as usize; if let Some(ref categories) = category_filter { - if !categories.contains(&state.poi_data.category[row]) { + if !categories.contains(state.poi_data.category.get(row)) { return None; } } @@ -83,11 +83,11 @@ pub async fn get_pois( .map(|&row| POI { id: state.poi_data.id[row].clone(), name: state.poi_data.name[row].clone(), - category: state.poi_data.category[row].clone(), - group: state.poi_data.group[row].clone(), + category: state.poi_data.category.get(row).to_string(), + group: state.poi_data.group.get(row).to_string(), lat: state.poi_data.lat[row], lng: state.poi_data.lng[row], - emoji: state.poi_data.emoji[row].clone(), + emoji: state.poi_data.emoji.get(row).to_string(), }) .collect(); diff --git a/server-rs/src/routes/properties.rs b/server-rs/src/routes/properties.rs index 8adffd7..8131ffe 100644 --- a/server-rs/src/routes/properties.rs +++ b/server-rs/src/routes/properties.rs @@ -8,7 +8,7 @@ use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; -use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, MAX_PROPERTIES_LIMIT}; +use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT}; use crate::data::EnumFeatureData; use crate::filter::{parse_filters, row_passes_filters}; use crate::state::AppState; @@ -36,13 +36,13 @@ pub struct Property { pub potential_energy_rating: Option, // Numeric fields - pub lat: f64, - pub lon: f64, + pub lat: f32, + pub lon: f32, pub is_construction_date_approximate: Option, #[serde(flatten)] - pub features: FxHashMap, + pub features: FxHashMap, } #[derive(Serialize)] @@ -93,17 +93,21 @@ pub async fn get_hexagon_properties( })?; let cell_u64: u64 = cell.into(); - let resolution = params.resolution as usize; - if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() { + let resolution = params.resolution; + if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) { warn!( resolution, - "Invalid or non-precomputed resolution for hexagon-properties" + "Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX ); return Err(( StatusCode::BAD_REQUEST, - "Invalid or non-precomputed resolution".to_string(), + format!( + "resolution must be between {} and {}", + H3_REQUEST_MIN, H3_REQUEST_MAX + ), )); } + let resolution_idx = resolution as usize; let h3_str = params.h3.clone(); let filters_str = params.filters.clone(); @@ -116,7 +120,13 @@ pub async fn get_hexagon_properties( let result = tokio::task::spawn_blocking(move || { let t0 = std::time::Instant::now(); - let h3_data = &state.h3_cells[resolution]; + let precomputed: Option<&[u64]> = state + .h3_cells + .get(resolution_idx) + .filter(|cells| !cells.is_empty()) + .map(|cells| cells.as_slice()); + let h3_res = h3o::Resolution::try_from(resolution) + .map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?; let num_features = state.data.num_features; let feature_data = &state.data.feature_data; let enum_features = &state.data.enum_features; @@ -128,7 +138,14 @@ pub async fn get_hexagon_properties( .grid .for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| { let row = row_idx as usize; - if h3_data[row] == cell_u64 + let row_cell = if let Some(h3_data) = precomputed { + h3_data[row] + } else { + h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) + .map(|coord| u64::from(coord.to_cell(h3_res))) + .unwrap_or(0) + }; + if row_cell == cell_u64 && row_passes_filters( row, &parsed_filters, @@ -162,8 +179,8 @@ pub async fn get_hexagon_properties( } Property { - address: non_empty_string(&state.data.address[row]), - postcode: non_empty_string(&state.data.postcode[row]), + address: non_empty_string(state.data.address(row)), + postcode: non_empty_string(state.data.postcode(row)), is_construction_date_approximate: Some(state.data.is_approx_build_date[row]), property_type: lookup_enum_value( enum_features, @@ -215,16 +232,17 @@ pub async fn get_hexagon_properties( "GET /api/hexagon-properties" ); - HexagonPropertiesResponse { + Ok(HexagonPropertiesResponse { properties, total, limit, offset, truncated, - } + }) }) .await - .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?; + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))? + .map_err(|error: String| (StatusCode::INTERNAL_SERVER_ERROR, error))?; Ok(Json(result)) } diff --git a/server-rs/src/tests.rs b/server-rs/src/tests.rs index 25df804..4728843 100644 --- a/server-rs/src/tests.rs +++ b/server-rs/src/tests.rs @@ -4,8 +4,8 @@ mod grid_index_tests { #[test] fn query_bounds_fully_below_grid_returns_empty() { - let lat = vec![50.0, 50.5, 51.0]; - let lon = vec![0.0, 0.5, 1.0]; + let lat = vec![50.0_f32, 50.5, 51.0]; + let lon = vec![0.0_f32, 0.5, 1.0]; let grid = GridIndex::build(&lat, &lon, 0.01); let results = grid.query(10.0, -10.0, 20.0, -5.0); @@ -17,8 +17,8 @@ mod grid_index_tests { #[test] fn query_bounds_fully_above_grid_returns_empty() { - let lat = vec![50.0, 50.5, 51.0]; - let lon = vec![0.0, 0.5, 1.0]; + let lat = vec![50.0_f32, 50.5, 51.0]; + let lon = vec![0.0_f32, 0.5, 1.0]; let grid = GridIndex::build(&lat, &lon, 0.01); let results = grid.query(80.0, 50.0, 90.0, 60.0); @@ -30,8 +30,8 @@ mod grid_index_tests { #[test] fn query_inverted_bounds_returns_empty() { - let lat = vec![50.0, 50.5, 51.0]; - let lon = vec![0.0, 0.5, 1.0]; + let lat = vec![50.0_f32, 50.5, 51.0]; + let lon = vec![0.0_f32, 0.5, 1.0]; let grid = GridIndex::build(&lat, &lon, 0.01); // south > north @@ -44,8 +44,8 @@ mod grid_index_tests { #[test] fn for_each_bounds_fully_outside_yields_nothing() { - let lat = vec![50.0, 50.5, 51.0]; - let lon = vec![0.0, 0.5, 1.0]; + let lat = vec![50.0_f32, 50.5, 51.0]; + let lon = vec![0.0_f32, 0.5, 1.0]; let grid = GridIndex::build(&lat, &lon, 0.01); let mut count = 0; @@ -60,8 +60,8 @@ mod grid_index_tests { fn query_with_large_cells_outside_returns_empty() { // Previously, out-of-bounds queries with large cell sizes would // scan cell (0,0) which could contain data. Now returns empty. - let lat = vec![50.0]; - let lon = vec![0.0]; + let lat = vec![50.0_f32]; + let lon = vec![0.0_f32]; let grid = GridIndex::build(&lat, &lon, 1.0); let results = grid.query(0.0, -50.0, 10.0, -40.0); @@ -73,8 +73,8 @@ mod grid_index_tests { #[test] fn query_within_bounds_returns_correct_results() { - let lat = vec![50.0, 50.5, 51.0]; - let lon = vec![0.0, 0.5, 1.0]; + let lat = vec![50.0_f32, 50.5, 51.0]; + let lon = vec![0.0_f32, 0.5, 1.0]; let grid = GridIndex::build(&lat, &lon, 0.01); let results = grid.query(49.9, -0.1, 51.1, 1.1); @@ -83,8 +83,8 @@ mod grid_index_tests { #[test] fn query_partial_bounds_returns_subset() { - let lat = vec![50.0, 51.0, 52.0]; - let lon = vec![0.0, 0.0, 0.0]; + let lat = vec![50.0_f32, 51.0, 52.0]; + let lon = vec![0.0_f32, 0.0, 0.0]; let grid = GridIndex::build(&lat, &lon, 0.01); let results = grid.query(49.9, -0.1, 50.1, 0.1); @@ -100,7 +100,7 @@ mod filter_tests { #[test] fn nan_rows_fail_numeric_filter_even_with_infinite_range() { let feature_names = vec!["price".to_string()]; - let feature_data = vec![f64::NAN]; + let feature_data = vec![f32::NAN]; let enum_features: Vec = vec![]; let (numeric, enums) = From 2c613dc0d14edff046bb2e33c0e162da49f02b66 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 2 Feb 2026 20:10:32 +0000 Subject: [PATCH 02/66] Optimise --- README.md | 7 +- frontend/src/App.tsx | 19 +- frontend/src/usePlausible.ts | 3 + frontend/webpack.config.js | 6 - server-rs/src/consts.rs | 1 - server-rs/src/filter.rs | 5 +- server-rs/src/grid_index.rs | 58 ++++-- server-rs/src/main.rs | 3 +- server-rs/src/routes/hexagon_stats.rs | 71 ++++--- server-rs/src/routes/hexagons.rs | 262 +++++++++++++++++--------- server-rs/src/routes/pois.rs | 61 +++--- server-rs/src/routes/properties.rs | 53 ++++-- server-rs/src/state.rs | 6 +- server-rs/src/tests.rs | 9 +- 14 files changed, 376 insertions(+), 188 deletions(-) diff --git a/README.md b/README.md index e63ab17..a1b785a 100644 --- a/README.md +++ b/README.md @@ -105,4 +105,9 @@ righmove lins - how to handle too many pois \ No newline at end of file + how to handle too many pois + + + fix zoopla links + + \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 84af9b7..971be4a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -600,6 +600,12 @@ export default function App() { bounds: boundsStr, }); if (filtersStr) params.set('filters', filtersStr); + // Only request data for the actively viewed feature (reduces bandwidth) + if (viewFeature) { + params.set('fields', viewFeature); + } else { + params.set('fields', ''); + } const res = await fetch(`${getApiBaseUrl()}/api/hexagons?${params}`, { signal: abortControllerRef.current.signal, }); @@ -619,7 +625,7 @@ export default function App() { clearTimeout(debounceRef.current); } }; - }, [resolution, bounds, filters, buildFilterParam]); + }, [resolution, bounds, filters, buildFilterParam, viewFeature]); // During slider drag, use the expanded dataset (without active feature filter) // so both narrowing and expanding are visible. Otherwise use server-filtered data. @@ -745,6 +751,8 @@ export default function App() { const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`; const params = new URLSearchParams({ resolution: resolution.toString(), bounds: boundsStr }); if (filtersStr) params.set('filters', filtersStr); + // Only request the dragged feature's data + params.set('fields', name); fetch(`${getApiBaseUrl()}/api/hexagons?${params}`, { signal: dragAbortRef.current.signal, @@ -786,7 +794,7 @@ export default function App() { }, []); const fetchHexagonStats = useCallback( - async (h3: string, res: number, signal?: AbortSignal) => { + async (h3: string, res: number, signal?: AbortSignal, fields?: string[]) => { const params = new URLSearchParams({ h3, resolution: res.toString(), @@ -805,6 +813,9 @@ export default function App() { .join(','); params.append('filters', filterStr); } + if (fields) { + params.set('fields', fields.join(',')); + } const response = await fetch(`${getApiBaseUrl()}/api/hexagon-stats?${params}`, { signal }); return (await response.json()) as HexagonStatsResponse; }, @@ -903,7 +914,9 @@ export default function App() { try { if (rightPaneTab === 'area') { setLoadingHoveredAreaStats(true); - const stats = await fetchHexagonStats(h3, resolution, signal); + // On hover, only fetch stats for features that have active filters + const hoverFields = Object.keys(filters); + const stats = await fetchHexagonStats(h3, resolution, signal, hoverFields.length > 0 ? hoverFields : undefined); if (!signal.aborted) setHoveredAreaStats(stats); } else if (rightPaneTab === 'properties') { const params = new URLSearchParams({ diff --git a/frontend/src/usePlausible.ts b/frontend/src/usePlausible.ts index c56107d..27a4d72 100644 --- a/frontend/src/usePlausible.ts +++ b/frontend/src/usePlausible.ts @@ -1,5 +1,6 @@ const DOMAIN = 'narrowit.schmelczer.dev'; const ENDPOINT = '/status'; +const IS_DEV = process.env.NODE_ENV !== 'production'; type EventOptions = { props?: Record; @@ -7,6 +8,8 @@ type EventOptions = { }; function sendEvent(name: string, options?: EventOptions) { + if (IS_DEV) return; + const payload: Record = { n: name, u: window.location.href, diff --git a/frontend/webpack.config.js b/frontend/webpack.config.js index f0c4082..15b9889 100644 --- a/frontend/webpack.config.js +++ b/frontend/webpack.config.js @@ -51,12 +51,6 @@ module.exports = (env, argv) => { context: ['/api'], target: 'http://localhost:8001', }, - { - context: ['/status'], - target: 'https://stats.schmelczer.dev', - changeOrigin: true, - pathRewrite: { '^/status': '/api/event' }, - }, ], }, }; diff --git a/server-rs/src/consts.rs b/server-rs/src/consts.rs index f376e1a..da867b9 100644 --- a/server-rs/src/consts.rs +++ b/server-rs/src/consts.rs @@ -1,6 +1,5 @@ pub const HISTOGRAM_BINS: usize = 100; -pub const H3_PRECOMPUTE_MIN: u8 = 7; pub const H3_PRECOMPUTE_MAX: u8 = 12; pub const H3_REQUEST_MIN: u8 = 4; pub const H3_REQUEST_MAX: u8 = 12; diff --git a/server-rs/src/filter.rs b/server-rs/src/filter.rs index 69d6e63..c868220 100644 --- a/server-rs/src/filter.rs +++ b/server-rs/src/filter.rs @@ -74,13 +74,14 @@ pub fn row_passes_filters( enum_filters: &[ParsedEnumFilter], feature_data: &[f32], num_features: usize, - enum_features: &[EnumFeatureData], + enum_data: &[u8], + num_enums: usize, ) -> bool { filters.iter().all(|filter| { let value = feature_data[row * num_features + filter.feat_idx]; value.is_finite() && value >= filter.min && value <= filter.max }) && enum_filters.iter().all(|enum_filter| { - let value = enum_features[enum_filter.enum_idx].data[row]; + let value = enum_data[row * num_enums + enum_filter.enum_idx]; value != ENUM_NULL && enum_filter.allowed.contains(&value) }) } diff --git a/server-rs/src/grid_index.rs b/server-rs/src/grid_index.rs index 6849cbe..e8d2967 100644 --- a/server-rs/src/grid_index.rs +++ b/server-rs/src/grid_index.rs @@ -1,15 +1,20 @@ /// Grid-based spatial index for fast rectangle queries over property rows. /// -/// Divides the UK bounding box into cells of ~0.01 degrees (~1km), -/// each storing indices of rows whose lat/lon falls within that cell. +/// Divides the bounding box into cells of ~0.01 degrees (~1km). +/// Uses a Compressed Sparse Row (CSR) layout: a single flat `values` array +/// plus an `offsets` array so that cell `i` owns `values[offsets[i]..offsets[i+1]]`. +/// This eliminates per-cell Vec overhead (24 bytes each for ptr+len+cap). pub struct GridIndex { min_lat: f32, min_lon: f32, cell_size: f32, cols: usize, rows: usize, - /// cells[row * cols + col] = vec of row indices - cells: Vec>, + /// Flat array of row indices, grouped by cell. + values: Vec, + /// offsets[i] is the start index in `values` for cell i. + /// offsets[num_cells] is values.len() (sentinel). + offsets: Vec, } impl GridIndex { @@ -41,25 +46,47 @@ impl GridIndex { let rows = ((max_lat - min_lat) / cell_size).ceil() as usize + 1; let cols = ((max_lon - min_lon) / cell_size).ceil() as usize + 1; + let num_cells = rows * cols; tracing::debug!( rows_grid = rows, cols_grid = cols, - total_cells = rows * cols, + total_cells = num_cells, cell_size, - "Building grid index" + "Building grid index (CSR)" ); - let mut cells: Vec> = vec![Vec::new(); rows * cols]; + // First pass: count items per cell + let mut counts = vec![0u32; num_cells]; + for index in 0..lat.len() { + let grid_row = ((lat[index] - min_lat) / cell_size) as usize; + let grid_col = ((lon[index] - min_lon) / cell_size) as usize; + counts[grid_row * cols + grid_col] += 1; + } + // Build offsets from counts (prefix sum) + let mut offsets = Vec::with_capacity(num_cells + 1); + let mut running = 0u32; + for &count in &counts { + offsets.push(running); + running += count; + } + offsets.push(running); + let total = running as usize; + + // Second pass: fill values using write cursors + let mut cursors = offsets[..num_cells].to_vec(); + let mut values = vec![0u32; total]; for index in 0..lat.len() { let grid_row = ((lat[index] - min_lat) / cell_size) as usize; let grid_col = ((lon[index] - min_lon) / cell_size) as usize; let cell_index = grid_row * cols + grid_col; - cells[cell_index].push(index as u32); + let pos = cursors[cell_index] as usize; + values[pos] = index as u32; + cursors[cell_index] += 1; } - tracing::debug!("Grid index built"); + tracing::debug!("Grid index built (CSR)"); GridIndex { min_lat, @@ -67,7 +94,8 @@ impl GridIndex { cell_size, cols, rows, - cells, + values, + offsets, } } @@ -83,7 +111,10 @@ impl GridIndex { for row in row_min..=row_max { let row_start = row * self.cols; for col in col_min..=col_max { - result.extend_from_slice(&self.cells[row_start + col]); + let cell_idx = row_start + col; + let start = self.offsets[cell_idx] as usize; + let end = self.offsets[cell_idx + 1] as usize; + result.extend_from_slice(&self.values[start..end]); } } @@ -108,7 +139,10 @@ impl GridIndex { for row in row_min..=row_max { let row_start = row * self.cols; for col in col_min..=col_max { - for &row_idx in &self.cells[row_start + col] { + let cell_idx = row_start + col; + let start = self.offsets[cell_idx] as usize; + let end = self.offsets[cell_idx + 1] as usize; + for &row_idx in &self.values[start..end] { callback(row_idx); } } diff --git a/server-rs/src/main.rs b/server-rs/src/main.rs index eb5d569..4d8b9cf 100644 --- a/server-rs/src/main.rs +++ b/server-rs/src/main.rs @@ -72,8 +72,7 @@ async fn main() -> anyhow::Result<()> { let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, consts::GRID_CELL_SIZE); info!( - "Precomputing H3 cells for resolutions {}-{}", - consts::H3_PRECOMPUTE_MIN, + "Precomputing H3 cells at resolution {}", consts::H3_PRECOMPUTE_MAX ); let h3_cells = data::precompute_h3(&property_data.lat, &property_data.lon)?; diff --git a/server-rs/src/routes/hexagon_stats.rs b/server-rs/src/routes/hexagon_stats.rs index 0f5182c..f05d652 100644 --- a/server-rs/src/routes/hexagon_stats.rs +++ b/server-rs/src/routes/hexagon_stats.rs @@ -8,7 +8,7 @@ use axum::response::IntoResponse; use serde::Deserialize; use tracing::{info, warn}; -use crate::consts::{ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, HISTOGRAM_BINS}; +use crate::consts::{ENUM_NULL, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN, HISTOGRAM_BINS}; use crate::filter::{parse_filters, row_passes_filters}; use crate::state::AppState; @@ -19,6 +19,10 @@ pub struct HexagonStatsParams { pub h3: String, pub resolution: u8, pub filters: Option, + /// Comma-separated feature names to include in stats response. + /// When present (even if empty), only listed features are computed. + /// When absent, all features are returned (backward compatible). + pub fields: Option, } pub async fn get_hexagon_stats( @@ -45,8 +49,6 @@ pub async fn get_hexagon_stats( ), )); } - let resolution_idx = resolution as usize; - let h3_str = params.h3.clone(); let filters_str = params.filters.clone(); let (parsed_filters, parsed_enum_filters) = parse_filters( @@ -56,42 +58,58 @@ pub async fn get_hexagon_stats( ); let num_filters = parsed_filters.len() + parsed_enum_filters.len(); + // Parse optional `fields` param into sets of feature names. + // None = include all, Some = only include listed features. + let field_set: Option> = params.fields.as_ref().map(|fields_str| { + fields_str + .split(',') + .map(|field| field.trim().to_string()) + .filter(|field| !field.is_empty()) + .collect() + }); + let result = tokio::task::spawn_blocking(move || { let start_time = std::time::Instant::now(); - let precomputed: Option<&[u64]> = state - .h3_cells - .get(resolution_idx) - .filter(|cells| !cells.is_empty()) - .map(|cells| cells.as_slice()); + let precomputed = &state.h3_cells; let h3_res = h3o::Resolution::try_from(resolution) .map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?; + let need_parent = resolution < H3_PRECOMPUTE_MAX; let num_features = state.data.num_features; + let num_enums = state.data.num_enums; let feature_data = &state.data.feature_data; + let enum_data = &state.data.enum_data; let enum_features = &state.data.enum_features; let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(cell, 0.001); + // Resolve cell at requested resolution from precomputed max-resolution cell + let cell_for_row = |row: usize| -> u64 { + let max_cell = precomputed[row]; + if !need_parent || max_cell == 0 { + return max_cell; + } + h3o::CellIndex::try_from(max_cell) + .ok() + .and_then(|ci| ci.parent(h3_res)) + .map(u64::from) + .unwrap_or(0) + }; + // Collect matching rows let mut matching_rows: Vec = Vec::new(); state .grid .for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| { let row = row_idx as usize; - let row_cell = if let Some(h3_data) = precomputed { - h3_data[row] - } else { - h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) - .map(|coord| u64::from(coord.to_cell(h3_res))) - .unwrap_or(0) - }; - if row_cell == cell_u64 + if cell_for_row(row) == cell_u64 && row_passes_filters( row, &parsed_filters, &parsed_enum_filters, feature_data, num_features, - enum_features, + enum_data, + num_enums, ) { matching_rows.push(row); @@ -109,6 +127,12 @@ pub async fn get_hexagon_stats( output.push_str(",\"numeric_features\":["); let mut first_numeric = true; for (feature_index, feature_name) in state.data.feature_names.iter().enumerate() { + // Skip features not in the requested set (when fields param is present) + if let Some(ref set) = field_set { + if !set.contains(feature_name.as_str()) { + continue; + } + } let global_stats = &state.data.feature_stats[feature_index]; let histogram_min = global_stats.histogram.min; let histogram_max = global_stats.histogram.max; @@ -178,15 +202,20 @@ pub async fn get_hexagon_stats( output.push_str("],\"enum_features\":["); let mut first_enum = true; for enum_feature in enum_features { + // Skip enum features not in the requested set + if let Some(ref set) = field_set { + if !set.contains(enum_feature.name.as_str()) { + continue; + } + } let enum_index = match state.enum_name_to_idx.get(&enum_feature.name) { Some(&index) => index, None => continue, }; - let enum_data = &state.data.enum_features[enum_index]; - let mut value_counts = vec![0u64; enum_data.values.len()]; + let mut value_counts = vec![0u64; enum_feature.values.len()]; for &row in &matching_rows { - let value = enum_data.data[row]; + let value = enum_data[row * num_enums + enum_index]; if value != ENUM_NULL && (value as usize) < value_counts.len() { value_counts[value as usize] += 1; } @@ -215,7 +244,7 @@ pub async fn get_hexagon_stats( output.push(','); } first_value = false; - write_json_string(&mut output, &enum_data.values[value_index]); + write_json_string(&mut output, &enum_feature.values[value_index]); write!(output, ":{}", count).unwrap(); } output.push_str("}}"); diff --git a/server-rs/src/routes/hexagons.rs b/server-rs/src/routes/hexagons.rs index a515825..ac3c9da 100644 --- a/server-rs/src/routes/hexagons.rs +++ b/server-rs/src/routes/hexagons.rs @@ -9,8 +9,8 @@ use serde::Deserialize; use tracing::{info, warn}; use crate::consts::{ - BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, - POSTCODE_MIN_RESOLUTION, + BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, + H3_REQUEST_MIN, POSTCODE_MIN_RESOLUTION, }; use crate::filter::parse_filters; use crate::state::AppState; @@ -39,16 +39,21 @@ pub struct HexagonParams { /// Comma-separated filters: `name:min:max,...` /// Rows must have non-NaN values within [min,max] for each filter. filters: Option, + /// Comma-separated feature names to include in min/max aggregation. + /// When present (even if empty), only listed features are aggregated and written. + /// When absent, all features are included (backward compatible). + fields: Option, } -/// Per-cell accumulator for aggregating features +/// Per-cell accumulator for aggregating features. +/// Uses Box<[T]> instead of Vec to avoid storing capacity (saves 8 bytes per field per cell). struct CellAgg { count: u32, - mins: Vec, - maxs: Vec, + mins: Box<[f32]>, + maxs: Box<[f32]>, /// Min/max ordinal indices for enum features (255 = no data yet) - enum_mins: Vec, - enum_maxs: Vec, + enum_mins: Box<[u8]>, + enum_maxs: Box<[u8]>, /// Most common postcode in this cell (only tracked at high resolutions) postcode: Option, postcode_count: u32, @@ -60,10 +65,10 @@ impl CellAgg { fn new(num_features: usize, num_enums: usize) -> Self { CellAgg { count: 0, - mins: vec![f32::INFINITY; num_features], - maxs: vec![f32::NEG_INFINITY; num_features], - enum_mins: vec![ENUM_NULL; num_enums], - enum_maxs: vec![0; num_enums], + mins: vec![f32::INFINITY; num_features].into_boxed_slice(), + maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(), + enum_mins: vec![ENUM_NULL; num_enums].into_boxed_slice(), + enum_maxs: vec![0; num_enums].into_boxed_slice(), postcode: None, postcode_count: 0, lat_sum: 0.0, @@ -93,9 +98,45 @@ impl CellAgg { /// Track min/max ordinal index for each enum feature in this cell. #[inline] - fn add_enums(&mut self, enum_features: &[crate::data::EnumFeatureData], row: usize) { - for (enum_index, enum_feature) in enum_features.iter().enumerate() { - let value = enum_feature.data[row]; + fn add_enums(&mut self, enum_data: &[u8], row: usize, num_enums: usize) { + let base = row * num_enums; + let row_slice = &enum_data[base..base + num_enums]; + for (enum_index, &value) in row_slice.iter().enumerate() { + if value != ENUM_NULL { + if self.enum_mins[enum_index] == ENUM_NULL || value < self.enum_mins[enum_index] { + self.enum_mins[enum_index] = value; + } + if value > self.enum_maxs[enum_index] { + self.enum_maxs[enum_index] = value; + } + } + } + } + + /// Add a row, only aggregating the features at the given indices. + #[inline] + fn add_row_selective(&mut self, feature_data: &[f32], row: usize, num_features: usize, indices: &[usize]) { + self.count += 1; + let base = row * num_features; + for &feat_index in indices { + let value = feature_data[base + feat_index]; + if value.is_finite() { + if value < self.mins[feat_index] { + self.mins[feat_index] = value; + } + if value > self.maxs[feat_index] { + self.maxs[feat_index] = value; + } + } + } + } + + /// Track min/max ordinal index for selected enum features only. + #[inline] + fn add_enums_selective(&mut self, enum_data: &[u8], row: usize, num_enums: usize, indices: &[usize]) { + let base = row * num_enums; + for &enum_index in indices { + let value = enum_data[base + enum_index]; if value != ENUM_NULL { if self.enum_mins[enum_index] == ENUM_NULL || value < self.enum_mins[enum_index] { self.enum_mins[enum_index] = value; @@ -142,6 +183,7 @@ pub(crate) fn write_json_escaped(buf: &mut String, text: &str) { /// Write the hexagons JSON response directly to a String buffer, /// avoiding serde_json::Value allocations entirely. +/// When `numeric_indices` / `enum_indices` are Some, only those features are written. #[allow(clippy::too_many_arguments)] fn write_hexagons_json( buf: &mut String, @@ -153,6 +195,8 @@ fn write_hexagons_json( enum_max_keys: &[String], num_enums: usize, include_postcode: bool, + numeric_indices: Option<&[usize]>, + enum_indices: Option<&[usize]>, ) { buf.push_str("{\"features\":["); let mut first = true; @@ -168,24 +212,49 @@ fn write_hexagons_json( let _ = write!(buf, "{{\"h3\":\"{}\",\"count\":{}", cell, aggregation.count); - for feat_index in 0..num_features { - if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite() { - let _ = write!( - buf, - ",\"{}\":{},\"{}\":{}", - min_keys[feat_index], aggregation.mins[feat_index], max_keys[feat_index], aggregation.maxs[feat_index] - ); + if let Some(indices) = numeric_indices { + for &feat_index in indices { + if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite() { + let _ = write!( + buf, + ",\"{}\":{},\"{}\":{}", + min_keys[feat_index], aggregation.mins[feat_index], max_keys[feat_index], aggregation.maxs[feat_index] + ); + } + } + } else { + for feat_index in 0..num_features { + if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite() { + let _ = write!( + buf, + ",\"{}\":{},\"{}\":{}", + min_keys[feat_index], aggregation.mins[feat_index], max_keys[feat_index], aggregation.maxs[feat_index] + ); + } } } - for enum_index in 0..num_enums { - if aggregation.enum_mins[enum_index] != ENUM_NULL { - let _ = write!( - buf, - ",\"{}\":{},\"{}\":{}", - enum_min_keys[enum_index], aggregation.enum_mins[enum_index], - enum_max_keys[enum_index], aggregation.enum_maxs[enum_index] - ); + if let Some(indices) = enum_indices { + for &enum_index in indices { + if aggregation.enum_mins[enum_index] != ENUM_NULL { + let _ = write!( + buf, + ",\"{}\":{},\"{}\":{}", + enum_min_keys[enum_index], aggregation.enum_mins[enum_index], + enum_max_keys[enum_index], aggregation.enum_maxs[enum_index] + ); + } + } + } else { + for enum_index in 0..num_enums { + if aggregation.enum_mins[enum_index] != ENUM_NULL { + let _ = write!( + buf, + ",\"{}\":{},\"{}\":{}", + enum_min_keys[enum_index], aggregation.enum_mins[enum_index], + enum_max_keys[enum_index], aggregation.enum_maxs[enum_index] + ); + } } } @@ -253,27 +322,48 @@ pub async fn get_hexagons( ); let num_filters = parsed_filters.len() + parsed_enum_filters.len(); + // Parse optional `fields` param into numeric and enum index sets. + // If `fields` is absent (None), all features are included. + // If `fields` is present (even empty string), only listed features are included. + let field_indices: Option<(Vec, Vec)> = params.fields.as_ref().map(|fields_str| { + let mut numeric_indices = Vec::new(); + let mut enum_indices = Vec::new(); + if !fields_str.is_empty() { + for name in fields_str.split(',') { + let name = name.trim(); + if name.is_empty() { + continue; + } + if let Some(idx) = state.data.feature_names.iter().position(|feat| feat == name) { + numeric_indices.push(idx); + } else if let Some(&idx) = state.enum_name_to_idx.get(name) { + enum_indices.push(idx); + } + } + } + (numeric_indices, enum_indices) + }); + let json_body = tokio::task::spawn_blocking(move || -> Result { let t0 = std::time::Instant::now(); let num_features = state.data.num_features; - let num_enums = state.data.enum_features.len(); + let num_enums = state.data.num_enums; let feature_data = &state.data.feature_data; + let enum_data = &state.data.enum_data; let min_keys = &state.min_keys; let max_keys = &state.max_keys; let enum_min_keys = &state.enum_min_keys; let enum_max_keys = &state.enum_max_keys; - let h3_cells_for_res: Option<&[u64]> = state - .h3_cells - .get(resolution as usize) - .filter(|cells| !cells.is_empty()) - .map(|cells| cells.as_slice()); + let h3_res = h3o::Resolution::try_from(resolution) + .map_err(|error| format!("Invalid H3 resolution {}: {}", resolution, error))?; + let precomputed = &state.h3_cells; + let need_parent = resolution < H3_PRECOMPUTE_MAX; let mut groups: FxHashMap = FxHashMap::default(); - let enum_features = &state.data.enum_features; let include_postcode = resolution >= POSTCODE_MIN_RESOLUTION; // Row-level filter check: numeric must be non-NaN and within [min, max], @@ -283,60 +373,58 @@ pub async fn get_hexagons( let value = feature_data[row * num_features + filter.feat_idx]; value.is_finite() && value >= filter.min && value <= filter.max }) && parsed_enum_filters.iter().all(|enum_filter| { - let value = enum_features[enum_filter.enum_idx].data[row]; + let value = enum_data[row * num_enums + enum_filter.enum_idx]; value != ENUM_NULL && enum_filter.allowed.contains(&value) }) }; - if let Some(precomputed) = h3_cells_for_res { - state - .grid - .for_each_in_bounds(south, west, north, east, |row_idx| { - let row = row_idx as usize; - if !row_passes(row) { - return; - } - let cell_id = precomputed[row]; - let aggregation = groups - .entry(cell_id) - .or_insert_with(|| CellAgg::new(num_features, num_enums)); - aggregation.add_row(feature_data, row, num_features); - aggregation.add_enums(enum_features, row); - if include_postcode { - aggregation.add_postcode( - state.data.postcode(row), - state.data.lat[row], - state.data.lon[row], - ); - } - }); - } else { - let h3_res = h3o::Resolution::try_from(resolution) - .map_err(|error| format!("Invalid H3 resolution {}: {}", resolution, error))?; - state - .grid - .for_each_in_bounds(south, west, north, east, |row_idx| { - let row = row_idx as usize; - if !row_passes(row) { - return; - } - let cell_id = h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) - .map(|coord| u64::from(coord.to_cell(h3_res))) - .unwrap_or(0); - let aggregation = groups - .entry(cell_id) - .or_insert_with(|| CellAgg::new(num_features, num_enums)); - aggregation.add_row(feature_data, row, num_features); - aggregation.add_enums(enum_features, row); - if include_postcode { - aggregation.add_postcode( - state.data.postcode(row), - state.data.lat[row], - state.data.lon[row], - ); - } - }); - } + // Choose aggregation strategy based on whether fields are specified + let has_selective = field_indices.is_some(); + let (sel_numeric, sel_enum) = field_indices.as_ref().map_or((&[][..], &[][..]), |(ni, ei)| (ni.as_slice(), ei.as_slice())); + + let aggregate_row = |groups: &mut FxHashMap, cell_id: u64, row: usize| { + let aggregation = groups + .entry(cell_id) + .or_insert_with(|| CellAgg::new(num_features, num_enums)); + if has_selective { + aggregation.add_row_selective(feature_data, row, num_features, sel_numeric); + aggregation.add_enums_selective(enum_data, row, num_enums, sel_enum); + } else { + aggregation.add_row(feature_data, row, num_features); + aggregation.add_enums(enum_data, row, num_enums); + } + if include_postcode { + aggregation.add_postcode( + state.data.postcode(row), + state.data.lat[row], + state.data.lon[row], + ); + } + }; + + // Resolve cell at requested resolution from precomputed max-resolution cell. + // For max resolution, use directly; for lower resolutions, derive parent. + let cell_for_row = |row: usize| -> u64 { + let max_cell = precomputed[row]; + if !need_parent || max_cell == 0 { + return max_cell; + } + h3o::CellIndex::try_from(max_cell) + .ok() + .and_then(|ci| ci.parent(h3_res)) + .map(u64::from) + .unwrap_or(0) + }; + + state + .grid + .for_each_in_bounds(south, west, north, east, |row_idx| { + let row = row_idx as usize; + if !row_passes(row) { + return; + } + aggregate_row(&mut groups, cell_for_row(row), row); + }); let t_agg = t0.elapsed(); @@ -351,6 +439,8 @@ pub async fn get_hexagons( enum_max_keys, num_enums, include_postcode, + field_indices.as_ref().map(|(ni, _)| ni.as_slice()), + field_indices.as_ref().map(|(_, ei)| ei.as_slice()), ); let t_total = t0.elapsed(); diff --git a/server-rs/src/routes/pois.rs b/server-rs/src/routes/pois.rs index f4569f3..8caee48 100644 --- a/server-rs/src/routes/pois.rs +++ b/server-rs/src/routes/pois.rs @@ -2,14 +2,14 @@ use std::sync::Arc; use axum::extract::Query; use axum::http::StatusCode; -use axum::response::Json; +use axum::response::{IntoResponse, Json}; use serde::{Deserialize, Serialize}; use tracing::info; use crate::consts::MAX_POIS_PER_REQUEST; -use crate::data::POI; use crate::state::{AppState, POICategoryGroup}; +use super::hexagons::write_json_escaped; use super::parse::parse_bounds; #[derive(Deserialize)] @@ -19,15 +19,10 @@ pub struct POIParams { categories: Option, } -#[derive(Serialize)] -pub struct POIsResponse { - pois: Vec, -} - pub async fn get_pois( state: Arc, Query(params): Query, -) -> Result, (StatusCode, String)> { +) -> Result { let bounds_str = params.bounds.ok_or(( StatusCode::BAD_REQUEST, "bounds parameter is required".into(), @@ -44,7 +39,7 @@ pub async fn get_pois( let num_categories = category_filter.as_ref().map(|cats| cats.len()).unwrap_or(0); - let result = tokio::task::spawn_blocking(move || { + let json_body = tokio::task::spawn_blocking(move || { let t0 = std::time::Instant::now(); let row_indices = state.poi_grid.query(south, west, north, east); @@ -64,36 +59,46 @@ pub async fn get_pois( .collect(); if matching_rows.len() > MAX_POIS_PER_REQUEST { - // Use a power-of-2 sampling step so each POI's inclusion depends - // only on its own priority hash, not on what other POIs are in - // the viewport. This prevents visible reshuffling when panning. let ratio = (matching_rows.len() / MAX_POIS_PER_REQUEST) as u32; let step = ratio.next_power_of_two(); let mask = step - 1; matching_rows.retain(|&row| state.poi_data.priority[row] & mask == 0); - // Statistical noise may leave us slightly over the limit if matching_rows.len() > MAX_POIS_PER_REQUEST { matching_rows.sort_unstable_by_key(|&row| state.poi_data.priority[row]); matching_rows.truncate(MAX_POIS_PER_REQUEST); } } - let pois: Vec = matching_rows - .iter() - .map(|&row| POI { - id: state.poi_data.id[row].clone(), - name: state.poi_data.name[row].clone(), - category: state.poi_data.category.get(row).to_string(), - group: state.poi_data.group.get(row).to_string(), - lat: state.poi_data.lat[row], - lng: state.poi_data.lng[row], - emoji: state.poi_data.emoji.get(row).to_string(), - }) - .collect(); + // Write JSON directly to string buffer, avoiding intermediate POI allocations + let mut buf = String::with_capacity(matching_rows.len() * 128); + buf.push_str("{\"pois\":["); + + for (i, &row) in matching_rows.iter().enumerate() { + if i > 0 { + buf.push(','); + } + buf.push_str("{\"id\":\""); + write_json_escaped(&mut buf, &state.poi_data.id[row]); + buf.push_str("\",\"name\":\""); + write_json_escaped(&mut buf, &state.poi_data.name[row]); + buf.push_str("\",\"category\":\""); + write_json_escaped(&mut buf, state.poi_data.category.get(row)); + buf.push_str("\",\"group\":\""); + write_json_escaped(&mut buf, state.poi_data.group.get(row)); + buf.push_str("\",\"lat\":"); + buf.push_str(&state.poi_data.lat[row].to_string()); + buf.push_str(",\"lng\":"); + buf.push_str(&state.poi_data.lng[row].to_string()); + buf.push_str(",\"emoji\":\""); + write_json_escaped(&mut buf, state.poi_data.emoji.get(row)); + buf.push_str("\"}"); + } + + buf.push_str("]}"); let elapsed = t0.elapsed(); info!( - results = pois.len(), + results = matching_rows.len(), candidates = row_indices.len(), categories = num_categories, categories_raw = categories_str.as_deref().unwrap_or("-"), @@ -101,12 +106,12 @@ pub async fn get_pois( "GET /api/pois" ); - POIsResponse { pois } + buf }) .await .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?; - Ok(Json(result)) + Ok(([("content-type", "application/json")], json_body)) } #[derive(Serialize)] diff --git a/server-rs/src/routes/properties.rs b/server-rs/src/routes/properties.rs index 8131ffe..47592eb 100644 --- a/server-rs/src/routes/properties.rs +++ b/server-rs/src/routes/properties.rs @@ -8,7 +8,7 @@ use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; -use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT}; +use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT}; use crate::data::EnumFeatureData; use crate::filter::{parse_filters, row_passes_filters}; use crate::state::AppState; @@ -65,6 +65,8 @@ fn non_empty_string(text: &str) -> Option { fn lookup_enum_value( enum_features: &[EnumFeatureData], + enum_data: &[u8], + num_enums: usize, enum_idx: &FxHashMap, row: usize, names: &[&str], @@ -72,7 +74,7 @@ fn lookup_enum_value( for name in names { if let Some(&feature_index) = enum_idx.get(*name) { let enum_feature = &enum_features[feature_index]; - let data_index = enum_feature.data[row]; + let data_index = enum_data[row * num_enums + feature_index]; if data_index != ENUM_NULL { if let Some(value) = enum_feature.values.get(data_index as usize) { return Some(value.clone()); @@ -107,8 +109,6 @@ pub async fn get_hexagon_properties( ), )); } - let resolution_idx = resolution as usize; - let h3_str = params.h3.clone(); let filters_str = params.filters.clone(); let (parsed_filters, parsed_enum_filters) = parse_filters( @@ -120,39 +120,44 @@ pub async fn get_hexagon_properties( let result = tokio::task::spawn_blocking(move || { let t0 = std::time::Instant::now(); - let precomputed: Option<&[u64]> = state - .h3_cells - .get(resolution_idx) - .filter(|cells| !cells.is_empty()) - .map(|cells| cells.as_slice()); + let precomputed = &state.h3_cells; let h3_res = h3o::Resolution::try_from(resolution) .map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?; + let need_parent = resolution < H3_PRECOMPUTE_MAX; let num_features = state.data.num_features; + let num_enums = state.data.num_enums; let feature_data = &state.data.feature_data; + let enum_data_flat = &state.data.enum_data; let enum_features = &state.data.enum_features; let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(cell, 0.001); + let cell_for_row = |row: usize| -> u64 { + let max_cell = precomputed[row]; + if !need_parent || max_cell == 0 { + return max_cell; + } + h3o::CellIndex::try_from(max_cell) + .ok() + .and_then(|ci| ci.parent(h3_res)) + .map(u64::from) + .unwrap_or(0) + }; + let mut matching_rows: Vec = Vec::new(); state .grid .for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| { let row = row_idx as usize; - let row_cell = if let Some(h3_data) = precomputed { - h3_data[row] - } else { - h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64) - .map(|coord| u64::from(coord.to_cell(h3_res))) - .unwrap_or(0) - }; - if row_cell == cell_u64 + if cell_for_row(row) == cell_u64 && row_passes_filters( row, &parsed_filters, &parsed_enum_filters, feature_data, num_features, - enum_features, + enum_data_flat, + num_enums, ) { matching_rows.push(row); @@ -181,33 +186,43 @@ pub async fn get_hexagon_properties( Property { address: non_empty_string(state.data.address(row)), postcode: non_empty_string(state.data.postcode(row)), - is_construction_date_approximate: Some(state.data.is_approx_build_date[row]), + is_construction_date_approximate: Some(state.data.is_approx_build_date(row)), property_type: lookup_enum_value( enum_features, + enum_data_flat, + num_enums, &state.enum_name_to_idx, row, &["Property type", "epc_property_type", "pp_property_type"], ), built_form: lookup_enum_value( enum_features, + enum_data_flat, + num_enums, &state.enum_name_to_idx, row, &["Property type/built form", "built_form"], ), duration: lookup_enum_value( enum_features, + enum_data_flat, + num_enums, &state.enum_name_to_idx, row, &["Leashold/Freehold", "duration"], ), current_energy_rating: lookup_enum_value( enum_features, + enum_data_flat, + num_enums, &state.enum_name_to_idx, row, &["Current energy rating", "current_energy_rating"], ), potential_energy_rating: lookup_enum_value( enum_features, + enum_data_flat, + num_enums, &state.enum_name_to_idx, row, &["Potential energy rating", "potential_energy_rating"], diff --git a/server-rs/src/state.rs b/server-rs/src/state.rs index 0c146fd..7152a66 100644 --- a/server-rs/src/state.rs +++ b/server-rs/src/state.rs @@ -13,9 +13,9 @@ pub struct POICategoryGroup { pub struct AppState { pub data: PropertyData, pub grid: GridIndex, - /// h3_cells[resolution][row_idx] = precomputed H3 cell ID. - /// Empty Vec for resolutions not precomputed. - pub h3_cells: Vec>, + /// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (12). + /// Parent cells for lower resolutions derived via CellIndex::parent(). + pub h3_cells: Vec, pub poi_data: POIData, pub poi_grid: GridIndex, /// Precomputed JSON key names: "min_{feature_name}" for each numeric feature diff --git a/server-rs/src/tests.rs b/server-rs/src/tests.rs index 4728843..467d265 100644 --- a/server-rs/src/tests.rs +++ b/server-rs/src/tests.rs @@ -102,12 +102,13 @@ mod filter_tests { let feature_names = vec!["price".to_string()]; let feature_data = vec![f32::NAN]; let enum_features: Vec = vec![]; + let enum_data: Vec = vec![]; let (numeric, enums) = parse_filters(Some("price:-inf:inf"), &feature_names, &enum_features); assert_eq!(numeric.len(), 1, "Should parse -inf:inf as valid filter"); - let passes = row_passes_filters(0, &numeric, &enums, &feature_data, 1, &enum_features); + let passes = row_passes_filters(0, &numeric, &enums, &feature_data, 1, &enum_data, 0); assert!(!passes, "NaN should fail filter even with infinite range"); } @@ -116,15 +117,16 @@ mod filter_tests { let enum_features = vec![EnumFeatureData { name: "rating".to_string(), values: vec!["A".to_string(), "B".to_string()], - data: vec![0], }]; let feature_names: Vec = vec![]; + // Row-major enum data: 1 row, 1 enum, value=0 (index into "A") + let enum_data: Vec = vec![0]; let (numeric, enums) = parse_filters(Some("rating:"), &feature_names, &enum_features); assert_eq!(enums.len(), 1); assert!(enums[0].allowed.is_empty()); - let passes = row_passes_filters(0, &numeric, &enums, &[], 0, &enum_features); + let passes = row_passes_filters(0, &numeric, &enums, &[], 0, &enum_data, 1); assert!(!passes, "Empty allowed set should reject all rows"); } @@ -133,7 +135,6 @@ mod filter_tests { let enum_features = vec![EnumFeatureData { name: "rating".to_string(), values: vec!["A".to_string(), "B".to_string()], - data: vec![0], }]; let feature_names: Vec = vec![]; From a677b9331f00f16a7296d9872b55ec9afe5c079a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 2 Feb 2026 21:56:35 +0000 Subject: [PATCH 03/66] Refactor --- frontend/src/App.tsx | 577 ++++-------------- frontend/src/components/AreaPane.tsx | 376 ++---------- frontend/src/components/DataSourcesPage.tsx | 20 +- frontend/src/components/DualHistogram.tsx | 109 ++++ frontend/src/components/EnumBarChart.tsx | 23 + .../src/components/ExternalSearchLinks.tsx | 53 ++ frontend/src/components/FAQPage.tsx | 4 +- frontend/src/components/Filters.tsx | 206 +++---- frontend/src/components/Header.tsx | 143 +++++ frontend/src/components/HexCanvas.tsx | 135 ++++ frontend/src/components/HomePage.tsx | 182 +----- frontend/src/components/InfoPopup.tsx | 55 ++ frontend/src/components/Map.tsx | 296 +-------- frontend/src/components/MapLegend.tsx | 66 ++ frontend/src/components/POIPane.tsx | 110 ++-- frontend/src/components/PostcodeSearch.tsx | 72 +++ frontend/src/components/PropertiesPane.tsx | 168 ++--- frontend/src/components/ui/label.tsx | 4 +- frontend/src/hooks/useClickOutside.ts | 13 + frontend/src/hooks/useFadeIn.ts | 21 + frontend/src/hooks/useTheme.ts | 27 + frontend/src/hooks/useUrlSync.ts | 37 ++ frontend/src/index.css | 5 +- frontend/src/lib/api.ts | 61 ++ frontend/src/lib/external-search.ts | 136 +++++ frontend/src/lib/format.ts | 24 + frontend/src/lib/map-utils.ts | 109 ++++ frontend/src/lib/url-state.ts | 113 ++++ 28 files changed, 1647 insertions(+), 1498 deletions(-) create mode 100644 frontend/src/components/DualHistogram.tsx create mode 100644 frontend/src/components/EnumBarChart.tsx create mode 100644 frontend/src/components/ExternalSearchLinks.tsx create mode 100644 frontend/src/components/Header.tsx create mode 100644 frontend/src/components/HexCanvas.tsx create mode 100644 frontend/src/components/InfoPopup.tsx create mode 100644 frontend/src/components/MapLegend.tsx create mode 100644 frontend/src/components/PostcodeSearch.tsx create mode 100644 frontend/src/hooks/useClickOutside.ts create mode 100644 frontend/src/hooks/useFadeIn.ts create mode 100644 frontend/src/hooks/useTheme.ts create mode 100644 frontend/src/hooks/useUrlSync.ts create mode 100644 frontend/src/lib/api.ts create mode 100644 frontend/src/lib/external-search.ts create mode 100644 frontend/src/lib/format.ts create mode 100644 frontend/src/lib/map-utils.ts create mode 100644 frontend/src/lib/url-state.ts diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 971be4a..39efee0 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -9,6 +9,7 @@ import DataSources from './components/DataSources'; import DataSourcesPage from './components/DataSourcesPage'; import FAQPage from './components/FAQPage'; import HomePage from './components/HomePage'; +import Header, { type Page } from './components/Header'; import type { FeatureMeta, FeatureGroup, @@ -21,321 +22,31 @@ import type { POIResponse, POICategoriesResponse, POICategoryGroup, - ViewState, Property, HexagonPropertiesResponse, HexagonStatsResponse, } from './types'; +import { fetchWithRetry, getApiBaseUrl, buildFilterString } from './lib/api'; +import { parseUrlState, DEFAULT_VIEW } from './lib/url-state'; +import { useTheme } from './hooks/useTheme'; +import { useUrlSync } from './hooks/useUrlSync'; -type Theme = 'light' | 'dark'; +declare global { + interface Window { + __og_ready?: boolean; + } +} const DEBOUNCE_MS = 150; -const URL_DEBOUNCE_MS = 300; -const INITIAL_RETRY_MS = 1000; -const MAX_RETRY_MS = 10000; - -async function fetchWithRetry( - url: string, - onSuccess: (data: T) => void, - signal: AbortSignal -): Promise { - let delay = INITIAL_RETRY_MS; - while (!signal.aborted) { - try { - const res = await fetch(url, { signal }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const json = await res.json(); - onSuccess(json); - return; - } catch (err) { - if (signal.aborted) return; - console.error(`Failed to fetch ${url}, retrying in ${delay}ms:`, err); - await new Promise((resolve) => setTimeout(resolve, delay)); - delay = Math.min(delay * 2, MAX_RETRY_MS); - } - } -} - -// Detect if running through VS Code web proxy and construct API base URL -function getApiBaseUrl(): string { - // In production builds, always use same-origin (Rust server serves both API and frontend) - if (process.env.NODE_ENV === 'production') { - return ''; - } - - const { pathname, href } = window.location; - - // Check pathname for /proxy/PORT pattern (VS Code web proxy) - const pathMatch = pathname.match(/^(\/proxy\/)(\d+)/); - if (pathMatch) { - return `${pathMatch[1]}8001`; - } - - // Check full href in case proxy rewrites pathname - const hrefMatch = href.match(/(\/proxy\/)\d+/); - if (hrefMatch) { - return `${hrefMatch[1]}8001`; - } - - // Default: same origin (works for local dev with webpack proxy) - return ''; -} - -const DEFAULT_VIEW: ViewState = { - longitude: -1.5, - latitude: 53.5, - zoom: 6, - pitch: 0, -}; - -// --- URL State helpers --- - -function parseUrlState(): { - viewState?: ViewState; - filters?: FeatureFilters; - poiCategories?: Set; - tab?: 'pois' | 'properties' | 'area'; -} { - const params = new URLSearchParams(window.location.search); - const result: ReturnType = {}; - - // Parse view: v=lat,lng,zoom - const v = params.get('v'); - if (v) { - const parts = v.split(',').map(Number); - if (parts.length === 3 && parts.every((n) => !isNaN(n))) { - result.viewState = { - latitude: parts[0], - longitude: parts[1], - zoom: parts[2], - pitch: 0, - }; - } - } - - // Parse filters: f=name:min:max,name:val1|val2 - const f = params.get('f'); - if (f) { - const filters: FeatureFilters = {}; - for (const segment of f.split(',')) { - const colonIdx = segment.indexOf(':'); - if (colonIdx === -1) continue; - const name = segment.substring(0, colonIdx); - const rest = segment.substring(colonIdx + 1); - if (rest.includes(':')) { - // Numeric: name:min:max - const [minStr, maxStr] = rest.split(':'); - const min = Number(minStr); - const max = Number(maxStr); - if (!isNaN(min) && !isNaN(max)) { - filters[name] = [min, max]; - } - } else if (rest.includes('|')) { - // Enum: name:val1|val2 - filters[name] = rest.split('|'); - } else { - // Single enum value - filters[name] = [rest]; - } - } - if (Object.keys(filters).length > 0) { - result.filters = filters; - } - } - - // Parse POI categories: poi=Cafe,Pub,School - const poi = params.get('poi'); - if (poi) { - result.poiCategories = new Set(poi.split(',').filter(Boolean)); - } - - // Parse tab: tab=p or tab=o or tab=a - const tab = params.get('tab'); - if (tab === 'p') result.tab = 'properties'; - else if (tab === 'o') result.tab = 'pois'; - else if (tab === 'a') result.tab = 'area'; - - return result; -} - -function stateToParams( - viewState: { latitude: number; longitude: number; zoom: number } | null, - filters: FeatureFilters, - features: FeatureMeta[], - selectedPOICategories: Set, - rightPaneTab: 'pois' | 'properties' | 'area' -): URLSearchParams { - const params = new URLSearchParams(); - - // View - if (viewState) { - params.set( - 'v', - `${viewState.latitude.toFixed(4)},${viewState.longitude.toFixed(4)},${viewState.zoom.toFixed(1)}` - ); - } - - // Filters - const filterEntries = Object.entries(filters); - if (filterEntries.length > 0) { - const filtersStr = filterEntries - .map(([name, value]) => { - const meta = features.find((f) => f.name === name); - if (meta?.type === 'enum') { - return `${name}:${(value as string[]).join('|')}`; - } - const [min, max] = value as [number, number]; - return `${name}:${min}:${max}`; - }) - .join(','); - params.set('f', filtersStr); - } - - // POI categories - if (selectedPOICategories.size > 0) { - params.set('poi', Array.from(selectedPOICategories).join(',')); - } - - // Tab (only if non-default) - if (rightPaneTab === 'properties') { - params.set('tab', 'p'); - } else if (rightPaneTab === 'area') { - params.set('tab', 'a'); - } - - return params; -} - -// --- Header --- - -type Page = 'home' | 'dashboard' | 'data-sources' | 'faq'; - -function Header({ - activePage, - onPageChange, - theme, - onToggleTheme, -}: { - activePage: Page; - onPageChange: (page: Page) => void; - theme: Theme; - onToggleTheme: () => void; -}) { - const [copied, setCopied] = useState(false); - - const handleShare = useCallback(() => { - navigator.clipboard.writeText(window.location.href).then(() => { - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }); - }, []); - - const tabClass = (page: Page) => - `px-3 py-1.5 rounded text-sm font-medium transition-colors ${ - activePage === page - ? 'bg-navy-700 text-white' - : 'text-warm-300 hover:bg-navy-800 hover:text-white' - }`; - - return ( -
-
- - -
-
- - {activePage === 'dashboard' && ( - - )} -
-
- ); -} - -// --- App --- export default function App() { - // Parse URL state once on mount const urlState = useMemo(() => parseUrlState(), []); + const isScreenshotMode = useMemo(() => { + const params = new URLSearchParams(window.location.search); + return params.get('screenshot') === '1'; + }, []); + const [features, setFeatures] = useState([]); const [filters, setFilters] = useState(urlState.filters || {}); const [activeFeature, setActiveFeature] = useState(null); @@ -351,7 +62,6 @@ export default function App() { const abortControllerRef = useRef(null); const dragAbortRef = useRef(null); - // View state for URL serialization const [currentView, setCurrentView] = useState<{ latitude: number; longitude: number; @@ -366,10 +76,8 @@ export default function App() { : null ); - // Initial view state for Map const initialViewState = useMemo(() => urlState.viewState || DEFAULT_VIEW, []); - // POI state const [pois, setPois] = useState([]); const [poiCategoryGroups, setPOICategoryGroups] = useState([]); const [selectedPOICategories, setSelectedPOICategories] = useState>( @@ -378,7 +86,6 @@ export default function App() { const poiDebounceRef = useRef | null>(null); const poiAbortControllerRef = useRef(null); - // Hexagon properties state const [selectedHexagon, setSelectedHexagon] = useState<{ h3: string; resolution: number } | null>( null ); @@ -386,13 +93,13 @@ export default function App() { const [propertiesTotal, setPropertiesTotal] = useState(0); const [propertiesOffset, setPropertiesOffset] = useState(0); const [loadingProperties, setLoadingProperties] = useState(false); - const [rightPaneTab, setRightPaneTab] = useState<'pois' | 'properties' | 'area'>(urlState.tab || 'pois'); + const [rightPaneTab, setRightPaneTab] = useState<'pois' | 'properties' | 'area'>( + urlState.tab || 'pois' + ); - // Area stats state const [areaStats, setAreaStats] = useState(null); const [loadingAreaStats, setLoadingAreaStats] = useState(false); - // Hover state const [hoveredHexagon, setHoveredHexagon] = useState(null); const [hoveredAreaStats, setHoveredAreaStats] = useState(null); const [hoveredProperties, setHoveredProperties] = useState(null); @@ -402,8 +109,9 @@ export default function App() { const hoverAbortRef = useRef(null); const hoverDebounceRef = useRef | null>(null); const [initialLoading, setInitialLoading] = useState(true); + const [activePage, setActivePage] = useState(() => { - // Restore from history state if available (e.g. back/forward navigation) + if (isScreenshotMode) return 'dashboard'; if (window.history.state?.page) return window.history.state.page; const params = new URLSearchParams(window.location.search); return params.has('v') || params.has('f') || params.has('poi') || params.has('tab') @@ -411,24 +119,21 @@ export default function App() { : 'home'; }); - // Feature name to auto-open in the info popup after back navigation const [pendingInfoFeature, setPendingInfoFeature] = useState(null); - // Navigate between pages with history support const navigateTo = useCallback((page: Page, hash?: string, infoFeature?: string) => { - // Before pushing, tag the current state with the info feature so back restores it if (infoFeature) { window.history.replaceState({ ...window.history.state, infoFeature }, ''); } - const url = hash ? `${window.location.pathname}${window.location.search}#${hash}` : `${window.location.pathname}${window.location.search}`; + const url = hash + ? `${window.location.pathname}${window.location.search}#${hash}` + : `${window.location.pathname}${window.location.search}`; window.history.pushState({ page }, '', url); setActivePage(page); trackPageview(); }, []); - // Handle browser back/forward useEffect(() => { - // Tag the initial state so popstate can restore it if (!window.history.state?.page) { window.history.replaceState({ page: activePage }, ''); } @@ -444,37 +149,19 @@ export default function App() { return () => window.removeEventListener('popstate', handlePopState); }, []); // eslint-disable-line react-hooks/exhaustive-deps - // Theme state — defaults to system preference on first visit - const [theme, setTheme] = useState(() => { - const stored = localStorage.getItem('theme'); - if (stored === 'light' || stored === 'dark') return stored; - return 'light'; - }); + const { theme, toggleTheme } = useTheme(); - // Sync dark class on and persist to localStorage useEffect(() => { - const root = document.documentElement; - if (theme === 'dark') { - root.classList.add('dark'); - } else { - root.classList.remove('dark'); + if (isScreenshotMode && !initialLoading && rawData.length > 0) { + window.__og_ready = true; } - localStorage.setItem('theme', theme); - }, [theme]); + }, [isScreenshotMode, initialLoading, rawData]); - const toggleTheme = useCallback(() => { - setTheme((prev) => (prev === 'light' ? 'dark' : 'light')); - }, []); - - // Derive enabled features from filter keys const enabledFeatures = useMemo(() => new Set(Object.keys(filters)), [filters]); - // Derive view feature: active drag takes priority over pinned const viewFeature = activeFeature || pinnedFeature; const viewSource: 'drag' | 'eye' | null = activeFeature ? 'drag' : pinnedFeature ? 'eye' : null; - // Color range: use the filter slider range when a numeric filter is active, - // otherwise fall back to the feature's full range from metadata. - // For enum features, use ordinal index range [0, values.length - 1]. + const colorRange = useMemo((): [number, number] | null => { if (!viewFeature) return null; const meta = features.find((f) => f.name === viewFeature); @@ -482,7 +169,6 @@ export default function App() { if (meta.type === 'enum' && meta.values && meta.values.length > 0) { return [0, meta.values.length - 1]; } - // Use live drag values or committed filter range if available if (activeFeature === viewFeature && dragValue) return dragValue; const filterVal = filters[viewFeature]; if (filterVal && typeof filterVal[0] === 'number') return filterVal as [number, number]; @@ -490,7 +176,6 @@ export default function App() { return null; }, [viewFeature, features, activeFeature, dragValue, filters]); - // Filter range: current drag or committed filter values, used for gray-out const filterRange = useMemo((): [number, number] | null => { if (!viewFeature) return null; if (activeFeature && dragValue) return dragValue; @@ -499,32 +184,8 @@ export default function App() { return null; }, [viewFeature, activeFeature, dragValue, filters]); - // --- URL sync --- - const urlDebounceRef = useRef | null>(null); + useUrlSync(currentView, filters, features, selectedPOICategories, rightPaneTab); - useEffect(() => { - if (urlDebounceRef.current) { - clearTimeout(urlDebounceRef.current); - } - urlDebounceRef.current = setTimeout(() => { - const params = stateToParams( - currentView, - filters, - features, - selectedPOICategories, - rightPaneTab - ); - const search = params.toString(); - const newUrl = search ? `${window.location.pathname}?${search}` : window.location.pathname; - window.history.replaceState({ ...window.history.state }, '', newUrl); - }, URL_DEBOUNCE_MS); - - return () => { - if (urlDebounceRef.current) clearTimeout(urlDebounceRef.current); - }; - }, [currentView, filters, features, selectedPOICategories, rightPaneTab]); - - // Fetch feature metadata + POI categories on mount with exponential backoff useEffect(() => { const controller = new AbortController(); let featuresLoaded = false; @@ -560,23 +221,11 @@ export default function App() { return () => controller.abort(); }, []); - // Build filter query string helper - const buildFilterParam = useCallback((): string => { - const filterEntries = Object.entries(filters); - if (filterEntries.length === 0) return ''; - return filterEntries - .map(([name, value]) => { - const meta = features.find((f) => f.name === name); - if (meta?.type === 'enum') { - return `${name}:${(value as string[]).join('|')}`; - } - const [min, max] = value as [number, number]; - return `${name}:${min}:${max}`; - }) - .join(','); - }, [filters, features]); + const buildFilterParam = useCallback( + (): string => buildFilterString(filters, features), + [filters, features] + ); - // Debounced fetch when resolution/bounds/filters change — always fetch hexagons useEffect(() => { if (!bounds) return; @@ -600,7 +249,6 @@ export default function App() { bounds: boundsStr, }); if (filtersStr) params.set('filters', filtersStr); - // Only request data for the actively viewed feature (reduces bandwidth) if (viewFeature) { params.set('fields', viewFeature); } else { @@ -627,11 +275,8 @@ export default function App() { }; }, [resolution, bounds, filters, buildFilterParam, viewFeature]); - // During slider drag, use the expanded dataset (without active feature filter) - // so both narrowing and expanding are visible. Otherwise use server-filtered data. const data = dragData ?? rawData; - // Fetch POIs when bounds or selected categories change useEffect(() => { if (!bounds || selectedPOICategories.size === 0) { setPois([]); @@ -683,7 +328,6 @@ export default function App() { latitude, longitude, }: ViewChangeParams) => { - // Only update bounds/resolution when quantized values actually change const boundsKey = `${newBounds.south},${newBounds.west},${newBounds.north},${newBounds.east},${newRes}`; if (boundsKey !== prevBoundsRef.current) { prevBoundsRef.current = boundsKey; @@ -725,12 +369,11 @@ export default function App() { const handleDragStart = useCallback( (name: string) => { const meta = features.find((f) => f.name === name); - if (meta?.type === 'enum') return; // No drag interaction for enum features + if (meta?.type === 'enum') return; setActiveFeature(name); const fval = filters[name]; setDragValue(fval && !Array.isArray(fval[0]) ? (fval as [number, number]) : null); - // Fetch hexagons without this feature's filter so we can expand the range if (!bounds) return; if (dragAbortRef.current) dragAbortRef.current.abort(); dragAbortRef.current = new AbortController(); @@ -751,7 +394,6 @@ export default function App() { const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`; const params = new URLSearchParams({ resolution: resolution.toString(), bounds: boundsStr }); if (filtersStr) params.set('filters', filtersStr); - // Only request the dragged feature's data params.set('fields', name); fetch(`${getApiBaseUrl()}/api/hexagons?${params}`, { @@ -799,20 +441,8 @@ export default function App() { h3, resolution: res.toString(), }); - const filterEntries = Object.entries(filters); - if (filterEntries.length > 0) { - const filterStr = filterEntries - .map(([name, value]) => { - const meta = features.find((feature) => feature.name === name); - if (meta?.type === 'enum') { - return `${name}:${(value as string[]).join('|')}`; - } - const [min, max] = value as [number, number]; - return `${name}:${min}:${max}`; - }) - .join(','); - params.append('filters', filterStr); - } + const filterStr = buildFilterString(filters, features); + if (filterStr) params.append('filters', filterStr); if (fields) { params.set('fields', fields.join(',')); } @@ -833,21 +463,8 @@ export default function App() { offset: offset.toString(), }); - // Add current filters - const filterEntries = Object.entries(filters); - if (filterEntries.length > 0) { - const filterStr = filterEntries - .map(([name, value]) => { - const meta = features.find((f) => f.name === name); - if (meta?.type === 'enum') { - return `${name}:${(value as string[]).join('|')}`; - } - const [min, max] = value as [number, number]; - return `${name}:${min}:${max}`; - }) - .join(','); - params.append('filters', filterStr); - } + const filterStr = buildFilterString(filters, features); + if (filterStr) params.append('filters', filterStr); const response = await fetch(`${getApiBaseUrl()}/api/hexagon-properties?${params}`); const data: HexagonPropertiesResponse = await response.json(); @@ -871,14 +488,13 @@ export default function App() { const handleHexagonClick = useCallback( (h3: string) => { if (selectedHexagon?.h3 === h3) { - // Deselect if clicking same hexagon setSelectedHexagon(null); setProperties([]); setAreaStats(null); } else { setSelectedHexagon({ h3, resolution }); setPropertiesOffset(0); - setRightPaneTab('area'); // Auto-switch to area tab + setRightPaneTab('area'); setLoadingAreaStats(true); fetchHexagonStats(h3, resolution) .then((stats) => setAreaStats(stats)) @@ -914,9 +530,13 @@ export default function App() { try { if (rightPaneTab === 'area') { setLoadingHoveredAreaStats(true); - // On hover, only fetch stats for features that have active filters const hoverFields = Object.keys(filters); - const stats = await fetchHexagonStats(h3, resolution, signal, hoverFields.length > 0 ? hoverFields : undefined); + const stats = await fetchHexagonStats( + h3, + resolution, + signal, + hoverFields.length > 0 ? hoverFields : undefined + ); if (!signal.aborted) setHoveredAreaStats(stats); } else if (rightPaneTab === 'properties') { const params = new URLSearchParams({ @@ -925,18 +545,8 @@ export default function App() { limit: '3', offset: '0', }); - const filterEntries = Object.entries(filters); - if (filterEntries.length > 0) { - const filterStr = filterEntries - .map(([name, value]) => { - const meta = features.find((feature) => feature.name === name); - if (meta?.type === 'enum') return `${name}:${(value as string[]).join('|')}`; - const [min, max] = value as [number, number]; - return `${name}:${min}:${max}`; - }) - .join(','); - params.append('filters', filterStr); - } + const filterStr = buildFilterString(filters, features); + if (filterStr) params.append('filters', filterStr); const response = await fetch(`${getApiBaseUrl()}/api/hexagon-properties?${params}`, { signal, }); @@ -978,9 +588,38 @@ export default function App() { setAreaStats(null); }, []); + if (isScreenshotMode) { + return ( +
+ {}} + onHexagonHover={() => {}} + initialViewState={initialViewState} + theme={theme} + /> +
+ ); + } + return (
-
+
{activePage === 'home' ? ( navigateTo('dashboard')} theme={theme} /> ) : activePage === 'data-sources' ? ( @@ -1065,7 +704,6 @@ export default function App() { navigateTo('data-sources')} />
- {/* Tab headers */}
- {/* Tab content */}
{rightPaneTab === 'area' ? ( { - const hexId = hoverMode && hoveredHexagon && hoveredHexagon !== selectedHexagon?.h3 + hexagonLocation={(() => { + const hexId = + hoverMode && hoveredHexagon && hoveredHexagon !== selectedHexagon?.h3 ? hoveredHexagon : selectedHexagon?.h3; - const hex = hexId ? data.find((d) => d.h3 === hexId) : null; - if (!hex || typeof hex.lat !== 'number' || typeof hex.lon !== 'number') return null; - return { - lat: hex.lat as number, - lon: hex.lon as number, - postcode: (hex.postcode as string | undefined) ?? null, - resolution, - }; - })() - } + const hex = hexId ? data.find((d) => d.h3 === hexId) : null; + if (!hex || typeof hex.lat !== 'number' || typeof hex.lon !== 'number') + return null; + return { + lat: hex.lat as number, + lon: hex.lon as number, + postcode: (hex.postcode as string | undefined) ?? null, + resolution, + }; + })()} filters={filters} /> ) : rightPaneTab === 'properties' ? ( @@ -1134,11 +786,20 @@ export default function App() { properties={hoverMode && hoveredProperties ? hoveredProperties : properties} total={hoverMode && hoveredProperties ? hoveredPropertiesTotal : propertiesTotal} loading={loadingProperties} - hexagonId={hoverMode && hoveredProperties ? hoveredHexagon : selectedHexagon?.h3 || null} + hexagonId={ + hoverMode && hoveredProperties ? hoveredHexagon : selectedHexagon?.h3 || null + } onLoadMore={handleLoadMoreProperties} onClose={handleCloseProperties} onNavigateToSource={(slug) => navigateTo('data-sources', slug)} - isHoveredPreview={!!(hoverMode && hoveredProperties && hoveredHexagon && hoveredHexagon !== selectedHexagon?.h3)} + isHoveredPreview={ + !!( + hoverMode && + hoveredProperties && + hoveredHexagon && + hoveredHexagon !== selectedHexagon?.h3 + ) + } hoverMode={hoverMode} onHoverModeChange={setHoverMode} /> diff --git a/frontend/src/components/AreaPane.tsx b/frontend/src/components/AreaPane.tsx index 3d551fb..88ba313 100644 --- a/frontend/src/components/AreaPane.tsx +++ b/frontend/src/components/AreaPane.tsx @@ -1,12 +1,10 @@ import { useMemo } from 'react'; import type { FeatureFilters, FeatureMeta, HexagonStatsResponse } from '../types'; - -interface HexagonLocation { - lat: number; - lon: number; - postcode: string | null; - resolution: number; -} +import type { HexagonLocation } from '../lib/external-search'; +import { formatValue } from '../lib/format'; +import { DualHistogram, LoadingSkeleton } from './DualHistogram'; +import EnumBarChart from './EnumBarChart'; +import ExternalSearchLinks from './ExternalSearchLinks'; interface AreaPaneProps { stats: HexagonStatsResponse | null; @@ -22,17 +20,7 @@ interface AreaPaneProps { filters: FeatureFilters; } -function formatValue(value: number): string { - if (Math.abs(value) >= 1_000_000) return `${(value / 1_000_000).toFixed(1)}M`; - if (Math.abs(value) >= 1_000) return `${(value / 1_000).toFixed(0)}k`; - if (Number.isInteger(value)) return value.toLocaleString(); - return value.toFixed(1); -} - -// Group features by their group field from globalFeatures -function groupFeatures( - globalFeatures: FeatureMeta[] -): { name: string; features: FeatureMeta[] }[] { +function groupFeatures(globalFeatures: FeatureMeta[]): { name: string; features: FeatureMeta[] }[] { const groups: { name: string; features: FeatureMeta[] }[] = []; const seen = new Set(); for (const feature of globalFeatures) { @@ -46,302 +34,6 @@ function groupFeatures( return groups; } -function downsampleBars(counts: number[], targetBars: number): number[] { - const step = Math.max(1, Math.floor(counts.length / targetBars)); - const bars: number[] = []; - for (let index = 0; index < counts.length; index += step) { - let sum = 0; - for (let offset = 0; offset < step && index + offset < counts.length; offset++) { - sum += counts[index + offset]; - } - bars.push(sum); - } - return bars; -} - -function DualHistogram({ - localCounts, - globalCounts, - min, - max, - globalMean, -}: { - localCounts: number[]; - globalCounts: number[]; - min: number; - max: number; - globalMean?: number; -}) { - const targetBars = 25; - const localBars = downsampleBars(localCounts, targetBars); - const globalBars = downsampleBars(globalCounts, targetBars); - - const barCount = Math.min(localBars.length, globalBars.length); - const localMax = Math.max(...localBars, 1); - const globalMax = Math.max(...globalBars, 1); - - const meanFraction = - globalMean != null && max > min ? (globalMean - min) / (max - min) : null; - - return ( -
-
- {Array.from({ length: barCount }).map((_, index) => { - const globalHeight = (globalBars[index] / globalMax) * 100; - const localHeight = (localBars[index] / localMax) * 100; - return ( -
-
-
0 ? 1 : 0.1, - }} - /> -
- ); - })} - {meanFraction != null && meanFraction >= 0 && meanFraction <= 1 && ( -
- )} -
-
- ); -} - -function SkeletonHistogram() { - return ( -
-
-
-
-
-
- {Array.from({ length: 15 }).map((_, i) => ( -
- ))} -
-
-
-
-
-
- ); -} - -function LoadingSkeleton() { - return ( -
- {[0, 1, 2].map((groupIdx) => ( -
-
-
- {Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => ( - - ))} -
-
- ))} -
- ); -} - -// Map app property types to each site's expected values -const PROPERTY_TYPE_MAP: Record = { - 'House': { rightmove: 'detached,semi-detached,terraced', onthemarket: 'property', zoopla: '' }, - 'Detached': { rightmove: 'detached', onthemarket: 'detached', zoopla: 'detached' }, - 'Semi-Detached': { rightmove: 'semi-detached', onthemarket: 'semi-detached', zoopla: 'semi_detached' }, - 'Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, - 'End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, - 'Enclosed Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, - 'Enclosed End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' }, - 'Flat': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' }, - 'Maisonette': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' }, - 'Bungalow': { rightmove: 'bungalow', onthemarket: 'bungalow', zoopla: 'bungalow' }, - 'Park home': { rightmove: 'park-home', onthemarket: 'property', zoopla: '' }, -}; - -// Approximate H3 hex edge length in miles by resolution -// See https://h3geo.org/docs/core-library/restable -const H3_RADIUS_MILES: Record = { - 4: 15, // ~24km edge → ~15mi - 5: 6, // ~9km → ~6mi - 6: 3, // ~3.5km → ~3mi - 7: 1, // ~1.3km → ~1mi - 8: 0.5, // ~0.5km → ~0.3mi, round up - 9: 0.25, // ~0.17km - 10: 0.25, // ~0.07km - 11: 0.25, // ~0.025km - 12: 0.25, -}; - -// Rightmove only accepts specific radius values -const RIGHTMOVE_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40]; -// OnTheMarket and Zoopla accept similar sets -const OTM_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40]; -const ZOOPLA_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 25, 30]; - -function nearestRadius(target: number, allowed: number[]): number { - return allowed.reduce((best, r) => (Math.abs(r - target) < Math.abs(best - target) ? r : best)); -} - -function buildPropertySearchUrls( - location: HexagonLocation, - filters: FeatureFilters -): { rightmove: string; onthemarket: string; zoopla: string } { - const { lat, lon, postcode, resolution } = location; - const radiusMiles = H3_RADIUS_MILES[resolution] ?? 1; - const coordStr = `${lat.toFixed(5)},${lon.toFixed(5)}`; - - // Extract price filters - const priceFilter = filters['Last known price']; - const minPrice = Array.isArray(priceFilter) && typeof priceFilter[0] === 'number' ? priceFilter[0] : undefined; - const maxPrice = Array.isArray(priceFilter) && typeof priceFilter[1] === 'number' ? priceFilter[1] : undefined; - - // Extract property type filters - const propertyTypes = filters['Property type']; - const selectedTypes = Array.isArray(propertyTypes) && typeof propertyTypes[0] === 'string' ? propertyTypes as string[] : []; - - // --- Rightmove --- - // Rightmove accepts both postcodes and lat,lon in searchLocation - const rmParams = new URLSearchParams(); - rmParams.set('searchLocation', postcode || coordStr); - rmParams.set('channel', 'BUY'); - rmParams.set('radius', String(nearestRadius(radiusMiles, RIGHTMOVE_RADII))); - if (minPrice !== undefined) rmParams.set('minPrice', String(Math.round(minPrice))); - if (maxPrice !== undefined) rmParams.set('maxPrice', String(Math.round(maxPrice))); - if (selectedTypes.length > 0) { - const rmTypes = [...new Set(selectedTypes.flatMap((t) => { - const mapped = PROPERTY_TYPE_MAP[t]?.rightmove; - return mapped ? mapped.split(',') : []; - }))]; - if (rmTypes.length > 0) rmParams.set('propertyTypes', rmTypes.join(',')); - } - const rightmove = `https://www.rightmove.co.uk/property-for-sale/find.html?${rmParams.toString()}`; - - // --- OnTheMarket --- - let otmType = 'property'; - if (selectedTypes.length > 0) { - const otmTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.onthemarket).filter(Boolean))]; - if (otmTypes.length === 1 && otmTypes[0] !== 'property') otmType = otmTypes[0]!; - } - const otmParams = new URLSearchParams(); - otmParams.set('radius', String(nearestRadius(radiusMiles, OTM_RADII))); - if (minPrice !== undefined) otmParams.set('min-price', String(Math.round(minPrice))); - if (maxPrice !== undefined) otmParams.set('max-price', String(Math.round(maxPrice))); - let onthemarket: string; - if (postcode) { - const slug = postcode.replace(/\s+/g, '-').toLowerCase(); - onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/${slug}/?${otmParams.toString()}`; - } else { - // Use lat/lon search with geo params for bigger hexagons without a postcode - otmParams.set('search-site', 'geo'); - otmParams.set('geo-lat', String(lat)); - otmParams.set('geo-lng', String(lon)); - onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/?${otmParams.toString()}`; - } - - // --- Zoopla --- - const zParams = new URLSearchParams(); - zParams.set('q', postcode || coordStr); - zParams.set('search_source', 'for-sale'); - zParams.set('radius', String(nearestRadius(radiusMiles, ZOOPLA_RADII))); - if (minPrice !== undefined) zParams.set('price_min', String(Math.round(minPrice))); - if (maxPrice !== undefined) zParams.set('price_max', String(Math.round(maxPrice))); - if (selectedTypes.length > 0) { - const zTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.zoopla).filter(Boolean))]; - for (const zt of zTypes) { - zParams.append('property_sub_type', zt!); - } - } - let zoopla: string; - if (postcode) { - const slug = postcode.replace(/\s+/g, '-').toLowerCase(); - zoopla = `https://www.zoopla.co.uk/for-sale/property/${slug}/?${zParams.toString()}`; - } else { - // Use coordinate-based path for bigger hexagons - zParams.set('geo_autocomplete_identifier', `geo_${lat}_${lon}`); - zoopla = `https://www.zoopla.co.uk/for-sale/property/?${zParams.toString()}`; - } - - return { rightmove, onthemarket, zoopla }; -} - -function ExternalSearchLinks({ location, filters }: { location: HexagonLocation; filters: FeatureFilters }) { - const urls = useMemo(() => buildPropertySearchUrls(location, filters), [location, filters]); - const radiusMiles = H3_RADIUS_MILES[location.resolution] ?? 1; - const label = location.postcode || `${radiusMiles}mi radius`; - - return ( -
-

- Search {label} on -

- -
- ); -} - -function EnumBarChart({ counts }: { counts: Record }) { - const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA); - const maxCount = Math.max(...entries.map(([, count]) => count), 1); - - return ( -
- {entries.map(([label, count]) => ( -
- - {label} - -
-
-
- {count} -
- ))} -
- ); -} - export default function AreaPane({ stats, globalFeatures, @@ -357,7 +49,6 @@ export default function AreaPane({ }: AreaPaneProps) { const featureGroups = useMemo(() => groupFeatures(globalFeatures), [globalFeatures]); - // Build lookup maps from stats const numericByName = useMemo(() => { if (!stats) return new Map(); return new Map(stats.numeric_features.map((feature) => [feature.name, feature])); @@ -368,7 +59,6 @@ export default function AreaPane({ return new Map(stats.enum_features.map((feature) => [feature.name, feature])); }, [stats]); - // Build lookup for global feature metadata (for histogram overlay) const globalFeatureByName = useMemo( () => new Map(globalFeatures.map((f) => [f.name, f])), [globalFeatures] @@ -384,7 +74,6 @@ export default function AreaPane({ return (
- {/* Header */}
@@ -403,18 +92,40 @@ export default function AreaPane({ ? 'text-teal-600 dark:text-teal-400 bg-teal-50 dark:bg-teal-900/30' : 'text-warm-400 hover:text-warm-700 dark:hover:text-warm-300' }`} - title={hoverMode ? 'Live preview on (click to lock)' : 'Live preview off (click to enable)'} + title={ + hoverMode ? 'Live preview on (click to lock)' : 'Live preview off (click to enable)' + } > - - - + + + @@ -435,17 +146,16 @@ export default function AreaPane({ )}
- {/* External search links */} - {hexagonLocation && stats && } + {hexagonLocation && stats && ( + + )} - {/* Stats content */}
{loading && !stats ? ( ) : stats ? (
{featureGroups.map((group) => { - // Check if any feature in this group has data const hasData = group.features.some( (feature) => numericByName.has(feature.name) || enumByName.has(feature.name) ); @@ -464,14 +174,14 @@ export default function AreaPane({ if (numericStats) { const globalFeature = globalFeatureByName.get(feature.name); const globalHistogram = globalFeature?.histogram; - // Compute a global mean from the global histogram for the mean line let globalMean: number | undefined; if (globalHistogram && globalHistogram.counts.length > 0) { const totalCount = globalHistogram.counts.reduce((a, b) => a + b, 0); if (totalCount > 0) { let weightedSum = 0; for (let i = 0; i < globalHistogram.counts.length; i++) { - const binCenter = globalHistogram.min + (i + 0.5) * globalHistogram.bin_width; + const binCenter = + globalHistogram.min + (i + 0.5) * globalHistogram.bin_width; weightedSum += binCenter * globalHistogram.counts[i]; } globalMean = weightedSum / totalCount; @@ -479,7 +189,10 @@ export default function AreaPane({ } return ( -
+
{feature.name} @@ -514,7 +227,10 @@ export default function AreaPane({ if (enumStats) { return ( -
+
{feature.name} diff --git a/frontend/src/components/DataSourcesPage.tsx b/frontend/src/components/DataSourcesPage.tsx index f836a26..f39b8a8 100644 --- a/frontend/src/components/DataSourcesPage.tsx +++ b/frontend/src/components/DataSourcesPage.tsx @@ -97,6 +97,14 @@ const DATA_SOURCES = [ url: 'https://www.ofcom.org.uk/phones-and-broadband/coverage-and-speeds/connected-nations-20252/data-downloads-2025', license: 'Open Government Licence v3.0', }, + { + id: 'council-tax', + name: 'Council Tax Levels 2025-26', + origin: 'Ministry of Housing, Communities & Local Government', + use: 'Annual council tax rates for Bands A-H for all 296 billing authorities in England, for a dwelling occupied by two adults. Joined to properties via local authority district code from the NSPL postcode lookup.', + url: 'https://www.gov.uk/government/statistics/council-tax-levels-set-by-local-authorities-in-england-2025-to-2026', + license: 'Open Government Licence v3.0', + }, ]; export default function DataSourcesPage() { @@ -135,7 +143,9 @@ export default function DataSourcesPage() {
{ cardRefs.current[source.id] = el; }} + ref={(el) => { + cardRefs.current[source.id] = el; + }} className={`bg-white dark:bg-navy-800 rounded-lg border p-5 ${ highlightedId === source.id ? 'border-teal-400 ring-2 ring-teal-400' @@ -143,12 +153,16 @@ export default function DataSourcesPage() { }`} >
-

{source.name}

+

+ {source.name} +

{source.license}
-

Source: {source.origin}

+

+ Source: {source.origin} +

{source.use}

min ? (globalMean - min) / (max - min) : null; + + return ( +
+
+ {Array.from({ length: barCount }).map((_, index) => { + const globalHeight = (globalBars[index] / globalMax) * 100; + const localHeight = (localBars[index] / localMax) * 100; + return ( +
+
+
0 ? 1 : 0.1, + }} + /> +
+ ); + })} + {meanFraction != null && meanFraction >= 0 && meanFraction <= 1 && ( +
+ )} +
+
+ ); +} + +export function SkeletonHistogram() { + return ( +
+
+
+
+
+
+ {Array.from({ length: 15 }).map((_, i) => ( +
+ ))} +
+
+
+
+
+
+ ); +} + +export function LoadingSkeleton() { + return ( +
+ {[0, 1, 2].map((groupIdx) => ( +
+
+
+ {Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => ( + + ))} +
+
+ ))} +
+ ); +} diff --git a/frontend/src/components/EnumBarChart.tsx b/frontend/src/components/EnumBarChart.tsx new file mode 100644 index 0000000..8b305ae --- /dev/null +++ b/frontend/src/components/EnumBarChart.tsx @@ -0,0 +1,23 @@ +export default function EnumBarChart({ counts }: { counts: Record }) { + const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA); + const maxCount = Math.max(...entries.map(([, count]) => count), 1); + + return ( +
+ {entries.map(([label, count]) => ( +
+ + {label} + +
+
+
+ {count} +
+ ))} +
+ ); +} diff --git a/frontend/src/components/ExternalSearchLinks.tsx b/frontend/src/components/ExternalSearchLinks.tsx new file mode 100644 index 0000000..2ae4bd2 --- /dev/null +++ b/frontend/src/components/ExternalSearchLinks.tsx @@ -0,0 +1,53 @@ +import { useMemo } from 'react'; +import type { FeatureFilters } from '../types'; +import { + buildPropertySearchUrls, + H3_RADIUS_MILES, + type HexagonLocation, +} from '../lib/external-search'; + +export default function ExternalSearchLinks({ + location, + filters, +}: { + location: HexagonLocation; + filters: FeatureFilters; +}) { + const urls = useMemo(() => buildPropertySearchUrls(location, filters), [location, filters]); + const radiusMiles = H3_RADIUS_MILES[location.resolution] ?? 1; + const label = location.postcode || `${radiusMiles}mi radius`; + + return ( + + ); +} diff --git a/frontend/src/components/FAQPage.tsx b/frontend/src/components/FAQPage.tsx index 36bd032..da6b582 100644 --- a/frontend/src/components/FAQPage.tsx +++ b/frontend/src/components/FAQPage.tsx @@ -29,7 +29,7 @@ const FAQ_ITEMS: FAQItem[] = [ { question: 'What does the eye icon do on a filter?', answer: - 'The eye icon pins a feature as the colour source for the hexagon layer. When pinned, hexagons are coloured by that feature\'s value range even when you are not actively dragging its slider. This lets you visualise one feature while filtering on others. Click the eye icon again to unpin.', + "The eye icon pins a feature as the colour source for the hexagon layer. When pinned, hexagons are coloured by that feature's value range even when you are not actively dragging its slider. This lets you visualise one feature while filtering on others. Click the eye icon again to unpin.", }, { question: 'How fresh is the data?', @@ -39,7 +39,7 @@ const FAQ_ITEMS: FAQItem[] = [ { question: 'How are EPC records matched to Land Registry sales?', answer: - 'EPC and Land Registry records don\'t share a common identifier, so they are fuzzy-joined by address within each postcode bucket. The pipeline uses token-sorted string similarity with special handling for numeric tokens (house numbers, flat numbers). Matches are assigned greedily from highest similarity score downward so each record is used at most once.', + "EPC and Land Registry records don't share a common identifier, so they are fuzzy-joined by address within each postcode bucket. The pipeline uses token-sorted string similarity with special handling for numeric tokens (house numbers, flat numbers). Matches are assigned greedily from highest similarity score downward so each record is used at most once.", }, { question: 'What are Points of Interest (POIs)?', diff --git a/frontend/src/components/Filters.tsx b/frontend/src/components/Filters.tsx index 6c8e396..33176ea 100644 --- a/frontend/src/components/Filters.tsx +++ b/frontend/src/components/Filters.tsx @@ -2,6 +2,8 @@ import { memo, useState, useRef, useCallback, useMemo, useEffect } from 'react'; import { Slider } from './ui/slider'; import { Label } from './ui/label'; import type { FeatureMeta, FeatureFilters } from '../types'; +import { formatFilterValue } from '../lib/format'; +import InfoPopup from './InfoPopup'; interface FiltersProps { features: FeatureMeta[]; @@ -39,68 +41,6 @@ function EyeIcon({ filled, className }: { filled: boolean; className?: string }) ); } -function InfoPopup({ - feature, - onClose, - onNavigateToSource, -}: { - feature: FeatureMeta; - onClose: () => void; - onNavigateToSource?: (slug: string, featureName: string) => void; -}) { - const popupRef = useRef(null); - - useEffect(() => { - function handleClickOutside(e: MouseEvent) { - if (popupRef.current && !popupRef.current.contains(e.target as Node)) { - onClose(); - } - } - document.addEventListener('mousedown', handleClickOutside); - return () => document.removeEventListener('mousedown', handleClickOutside); - }, [onClose]); - - return ( -
-
-
-

- {feature.name} -

- -
- {feature.description && ( -

{feature.description}

- )} - {feature.detail && ( -

{feature.detail}

- )} - {feature.source && onNavigateToSource && ( - - )} -
-
- ); -} - function FeatureBrowser({ availableFeatures, allFeatures, @@ -123,7 +63,6 @@ function FeatureBrowser({ const [search, setSearch] = useState(''); const [infoFeature, setInfoFeature] = useState(null); - // Auto-open info popup when navigating back useEffect(() => { if (openInfoFeature) { const feat = allFeatures.find((f) => f.name === openInfoFeature); @@ -181,7 +120,9 @@ function FeatureBrowser({
{f.name} {f.description && ( - {f.description} + + {f.description} + )}
@@ -191,7 +132,13 @@ function FeatureBrowser({ className="text-warm-400 hover:text-warm-700 dark:hover:text-warm-300 p-0.5 rounded" title="Feature info" > - + @@ -209,7 +156,13 @@ function FeatureBrowser({ className="text-warm-400 hover:text-warm-700 dark:hover:text-warm-300 p-0.5 rounded" title="Add filter" > - + @@ -227,22 +180,36 @@ function FeatureBrowser({
{infoFeature && ( setInfoFeature(null)} - onNavigateToSource={onNavigateToSource} - /> + sourceLink={ + infoFeature.source && onNavigateToSource + ? { + label: 'View data source', + onClick: () => { + onNavigateToSource(infoFeature.source!, infoFeature.name); + setInfoFeature(null); + }, + } + : undefined + } + > + {infoFeature.description && ( +

+ {infoFeature.description} +

+ )} + {infoFeature.detail && ( +

+ {infoFeature.detail} +

+ )} + )} ); } -function formatValue(value: number): string { - if (Math.abs(value) >= 1_000_000) return `${(value / 1_000_000).toFixed(1)}M`; - if (Math.abs(value) >= 1_000) return `${(value / 1_000).toFixed(1)}k`; - if (Number.isInteger(value)) return value.toString(); - return value.toFixed(2); -} - export default memo(function Filters({ features, filters, @@ -258,7 +225,7 @@ export default memo(function Filters({ zoom, pinnedFeature, onTogglePin, - onCancelPin, + onCancelPin: _onCancelPin, onNavigateToSource, openInfoFeature, onClearOpenInfoFeature, @@ -270,38 +237,35 @@ export default memo(function Filters({ const [splitFraction, setSplitFraction] = useState(0.65); const draggingRef = useRef(false); - const handleSeparatorPointerDown = useCallback( - (e: React.PointerEvent) => { - e.preventDefault(); - (e.target as HTMLElement).setPointerCapture(e.pointerId); - draggingRef.current = true; - }, - [] - ); + const handleSeparatorPointerDown = useCallback((e: React.PointerEvent) => { + e.preventDefault(); + (e.target as HTMLElement).setPointerCapture(e.pointerId); + draggingRef.current = true; + }, []); - const handleSeparatorPointerMove = useCallback( - (e: React.PointerEvent) => { - if (!draggingRef.current || !containerRef.current) return; - const rect = containerRef.current.getBoundingClientRect(); - const y = e.clientY - rect.top; - const fraction = Math.min(0.8, Math.max(0.15, y / rect.height)); - setSplitFraction(fraction); - }, - [] - ); + const handleSeparatorPointerMove = useCallback((e: React.PointerEvent) => { + if (!draggingRef.current || !containerRef.current) return; + const rect = containerRef.current.getBoundingClientRect(); + const y = e.clientY - rect.top; + const fraction = Math.min(0.8, Math.max(0.15, y / rect.height)); + setSplitFraction(fraction); + }, []); const handleSeparatorPointerUp = useCallback(() => { draggingRef.current = false; }, []); return ( -
- {/* Top: Active filters — user-resizable, scrollable */} +
- {/* Active Filters header */}
- Active Filters + + Active Filters + {enabledFeatureList.length > 0 && ( {enabledFeatureList.length} @@ -314,11 +278,25 @@ export default memo(function Filters({
{enabledFeatureList.length === 0 && (
- - + + - No active filters - Browse features below and click + to add a filter + + No active filters + + + Browse features below and click + to add a filter +
)} @@ -327,14 +305,21 @@ export default memo(function Filters({ const selectedValues = (filters[feature.name] as string[]) || []; const allValues = feature.values || []; return ( -
+
@@ -363,7 +348,10 @@ export default memo(function Filters({
{allValues.map((val) => ( -