Optimisations
This commit is contained in:
parent
66c2a25457
commit
9179acd4cd
21 changed files with 653 additions and 139 deletions
65
README.md
65
README.md
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
```sh
|
||||
curl -1sLf 'https://dl.cloudsmith.io/public/task/task/setup.deb.sh' | sudo -E bash
|
||||
apt install task
|
||||
task prepare
|
||||
```
|
||||
|
||||
|
|
@ -34,30 +35,14 @@ task prepare
|
|||
5. fibre optic availability
|
||||
|
||||
6. between london and bournemouth ish
|
||||
|
||||
7. [Y] historical prices
|
||||
|
||||
8. current listings
|
||||
|
||||
## Action plan
|
||||
|
||||
1. use openstreetmap api to get the map
|
||||
|
||||
## Data Sources
|
||||
|
||||
- [Price Paid](https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads)
|
||||
|
||||
- [English Indices of Deprevation 2025](https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025)
|
||||
- The English Indices of Deprivation (IoD25) measure relative levels of deprivation in 33,755 small areas or neighbourhoods, called Lower-layer Super Output Areas (LSOAs), in England.
|
||||
|
||||
- [Population by Ethnicity and Region 2021](https://www.ethnicity-facts-figures.service.gov.uk/uk-population-by-ethnicity/national-and-regional-populations/regional-ethnic-diversity/latest/#download-the-data)
|
||||
|
||||
- [Crime](https://data.police.uk/data/)
|
||||
|
||||
- [Postcode -> GPS](https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data)
|
||||
|
||||
Nice to haves?
|
||||
|
||||
- <https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025> - file 8!
|
||||
|
||||
## Backend Data Sources
|
||||
|
|
@ -70,10 +55,54 @@ Nice to haves?
|
|||
- [Open Geography](https://geoportal.statistics.gov.uk/)
|
||||
- [CommunitiesOpenData](https://communitiesopendata-communities.hub.arcgis.com/)
|
||||
- [PlanetOSM](https://planet.openstreetmap.org/) for open street map POI
|
||||
- [TFL api](https://api-portal.tfl.gov.uk/signin)
|
||||
- [EPC](https://epc.opendatacommunities.org/login) - <https://epc.opendatacommunities.org/downloads/domestic>
|
||||
- [naptan](https://naptan.dft.gov.uk/naptan/schema/2.4/doc/NaPTANSchemaGuide-2.4-v0.57.pdf)
|
||||
|
||||
|
||||
rightmove:
|
||||
curl '<https://www.rightmove.co.uk/api/property-search/listing/search?searchLocation=E14&useLocationIdentifier=true&locationIdentifier=OUTCODE%5E749&buy=For+sale&radius=20.0&_includeSSTC=on&index=0&sortType=2&channel=BUY&transactionType=BUY>'
|
||||
|
||||
curl '<https://www.onthemarket.com/async/search/properties-v2/?search-type=for-sale&location-id=e13&view=map-list>'
|
||||
|
||||
|
||||
|
||||
Make it mobile friendly
|
||||
|
||||
Serve the frontend from the server
|
||||
|
||||
Add an account management page
|
||||
|
||||
|
||||
|
||||
categories the categories
|
||||
|
||||
|
||||
mkdir -p data/crime
|
||||
unzip data/d29f0314840ef7dcbb5cde66e383fe08059dab5a.zip -d data/crime/
|
||||
rm data/d29f0314840ef7dcbb5cde66e383fe08059dab5a.zip
|
||||
|
||||
|
||||
https://xploria.co.uk/data-sources/
|
||||
|
||||
|
||||
panning is slow
|
||||
|
||||
|
||||
|
||||
epc oopt out https://www.gov.uk/guidance/energy-performance-certificates-opt-out-of-public-disclosure
|
||||
|
||||
righmove lins
|
||||
|
||||
❯ Use the approach you described. Look at the current state of the frontend and backend and add user management to it. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be
|
||||
prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. There must be a use page showing the lifetime
|
||||
license key. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring
|
||||
of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too. Add a docker compose file to host pocketbase and the rust server. While doing this, protect the
|
||||
server against DOS-ing. Once you're logged in, you can save your searches and then look at your saved searches. Instead of just a share button in the header, add a save search one for logged in users and a login button (on all pages) for not-logged in users. Don't
|
||||
support profile pictures or full names to avoid GDPR requirements.
|
||||
|
||||
|
||||
|
||||
embedd google street view
|
||||
|
||||
|
||||
|
||||
how to handle too many pois
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
|
||||
import { trackPageview } from './usePlausible';
|
||||
import Map from './components/Map';
|
||||
import Filters from './components/Filters';
|
||||
import POIPane from './components/POIPane';
|
||||
|
|
@ -57,6 +58,11 @@ async function fetchWithRetry<T>(
|
|||
|
||||
// Detect if running through VS Code web proxy and construct API base URL
|
||||
function getApiBaseUrl(): string {
|
||||
// In production builds, always use same-origin (Rust server serves both API and frontend)
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
return '';
|
||||
}
|
||||
|
||||
const { pathname, href } = window.location;
|
||||
|
||||
// Check pathname for /proxy/PORT pattern (VS Code web proxy)
|
||||
|
|
@ -71,7 +77,7 @@ function getApiBaseUrl(): string {
|
|||
return `${hrefMatch[1]}8001`;
|
||||
}
|
||||
|
||||
// Default: same origin (works for both local dev with webpack proxy and production)
|
||||
// Default: same origin (works for local dev with webpack proxy)
|
||||
return '';
|
||||
}
|
||||
|
||||
|
|
@ -417,6 +423,7 @@ export default function App() {
|
|||
const url = hash ? `${window.location.pathname}${window.location.search}#${hash}` : `${window.location.pathname}${window.location.search}`;
|
||||
window.history.pushState({ page }, '', url);
|
||||
setActivePage(page);
|
||||
trackPageview();
|
||||
}, []);
|
||||
|
||||
// Handle browser back/forward
|
||||
|
|
@ -465,8 +472,9 @@ export default function App() {
|
|||
// Derive view feature: active drag takes priority over pinned
|
||||
const viewFeature = activeFeature || pinnedFeature;
|
||||
const viewSource: 'drag' | 'eye' | null = activeFeature ? 'drag' : pinnedFeature ? 'eye' : null;
|
||||
// Color range: always the feature's full slider range from metadata
|
||||
// For enum features, use ordinal index range [0, values.length - 1]
|
||||
// Color range: use the filter slider range when a numeric filter is active,
|
||||
// otherwise fall back to the feature's full range from metadata.
|
||||
// For enum features, use ordinal index range [0, values.length - 1].
|
||||
const colorRange = useMemo((): [number, number] | null => {
|
||||
if (!viewFeature) return null;
|
||||
const meta = features.find((f) => f.name === viewFeature);
|
||||
|
|
@ -474,9 +482,13 @@ export default function App() {
|
|||
if (meta.type === 'enum' && meta.values && meta.values.length > 0) {
|
||||
return [0, meta.values.length - 1];
|
||||
}
|
||||
// Use live drag values or committed filter range if available
|
||||
if (activeFeature === viewFeature && dragValue) return dragValue;
|
||||
const filterVal = filters[viewFeature];
|
||||
if (filterVal && typeof filterVal[0] === 'number') return filterVal as [number, number];
|
||||
if (meta.min != null && meta.max != null) return [meta.min, meta.max];
|
||||
return null;
|
||||
}, [viewFeature, features]);
|
||||
}, [viewFeature, features, activeFeature, dragValue, filters]);
|
||||
|
||||
// Filter range: current drag or committed filter values, used for gray-out
|
||||
const filterRange = useMemo((): [number, number] | null => {
|
||||
|
|
@ -1087,6 +1099,22 @@ export default function App() {
|
|||
onHoverModeChange={setHoverMode}
|
||||
onViewProperties={handleViewPropertiesFromArea}
|
||||
onClose={handleCloseProperties}
|
||||
hexagonLocation={
|
||||
(() => {
|
||||
const hexId = hoverMode && hoveredHexagon && hoveredHexagon !== selectedHexagon?.h3
|
||||
? hoveredHexagon
|
||||
: selectedHexagon?.h3;
|
||||
const hex = hexId ? data.find((d) => d.h3 === hexId) : null;
|
||||
if (!hex || typeof hex.lat !== 'number' || typeof hex.lon !== 'number') return null;
|
||||
return {
|
||||
lat: hex.lat as number,
|
||||
lon: hex.lon as number,
|
||||
postcode: (hex.postcode as string | undefined) ?? null,
|
||||
resolution,
|
||||
};
|
||||
})()
|
||||
}
|
||||
filters={filters}
|
||||
/>
|
||||
) : rightPaneTab === 'properties' ? (
|
||||
<PropertiesPane
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
import { useMemo } from 'react';
|
||||
import type { FeatureMeta, HexagonStatsResponse } from '../types';
|
||||
import type { FeatureFilters, FeatureMeta, HexagonStatsResponse } from '../types';
|
||||
|
||||
interface HexagonLocation {
|
||||
lat: number;
|
||||
lon: number;
|
||||
postcode: string | null;
|
||||
resolution: number;
|
||||
}
|
||||
|
||||
interface AreaPaneProps {
|
||||
stats: HexagonStatsResponse | null;
|
||||
|
|
@ -11,6 +18,8 @@ interface AreaPaneProps {
|
|||
onHoverModeChange: (enabled: boolean) => void;
|
||||
onViewProperties: () => void;
|
||||
onClose: () => void;
|
||||
hexagonLocation: HexagonLocation | null;
|
||||
filters: FeatureFilters;
|
||||
}
|
||||
|
||||
function formatValue(value: number): string {
|
||||
|
|
@ -37,10 +46,7 @@ function groupFeatures(
|
|||
return groups;
|
||||
}
|
||||
|
||||
function MiniHistogram({ counts, maxCount }: { counts: number[]; maxCount: number }) {
|
||||
if (maxCount === 0) return null;
|
||||
// Downsample to ~20 bars for display
|
||||
const targetBars = 20;
|
||||
function downsampleBars(counts: number[], targetBars: number): number[] {
|
||||
const step = Math.max(1, Math.floor(counts.length / targetBars));
|
||||
const bars: number[] = [];
|
||||
for (let index = 0; index < counts.length; index += step) {
|
||||
|
|
@ -50,21 +56,268 @@ function MiniHistogram({ counts, maxCount }: { counts: number[]; maxCount: numbe
|
|||
}
|
||||
bars.push(sum);
|
||||
}
|
||||
const barMax = Math.max(...bars, 1);
|
||||
return bars;
|
||||
}
|
||||
|
||||
function DualHistogram({
|
||||
localCounts,
|
||||
globalCounts,
|
||||
min,
|
||||
max,
|
||||
globalMean,
|
||||
}: {
|
||||
localCounts: number[];
|
||||
globalCounts: number[];
|
||||
min: number;
|
||||
max: number;
|
||||
globalMean?: number;
|
||||
}) {
|
||||
const targetBars = 25;
|
||||
const localBars = downsampleBars(localCounts, targetBars);
|
||||
const globalBars = downsampleBars(globalCounts, targetBars);
|
||||
|
||||
const barCount = Math.min(localBars.length, globalBars.length);
|
||||
const localMax = Math.max(...localBars, 1);
|
||||
const globalMax = Math.max(...globalBars, 1);
|
||||
|
||||
const meanFraction =
|
||||
globalMean != null && max > min ? (globalMean - min) / (max - min) : null;
|
||||
|
||||
return (
|
||||
<div className="flex items-end gap-px h-8 mt-1">
|
||||
{bars.map((count, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className="flex-1 bg-teal-500 dark:bg-teal-400 rounded-t-sm min-w-[2px]"
|
||||
style={{ height: `${(count / barMax) * 100}%`, opacity: count > 0 ? 1 : 0.1 }}
|
||||
/>
|
||||
<div className="mt-1">
|
||||
<div className="relative flex items-end gap-px h-10">
|
||||
{Array.from({ length: barCount }).map((_, index) => {
|
||||
const globalHeight = (globalBars[index] / globalMax) * 100;
|
||||
const localHeight = (localBars[index] / localMax) * 100;
|
||||
return (
|
||||
<div key={index} className="flex-1 relative min-w-[2px] h-full flex items-end">
|
||||
<div
|
||||
className="absolute bottom-0 left-0 right-0 bg-warm-300/40 dark:bg-warm-600/40 rounded-t-sm"
|
||||
style={{ height: `${globalHeight}%` }}
|
||||
/>
|
||||
<div
|
||||
className="absolute bottom-0 left-0 right-0 bg-teal-500 dark:bg-teal-400 rounded-t-sm"
|
||||
style={{
|
||||
height: `${localHeight}%`,
|
||||
opacity: localBars[index] > 0 ? 1 : 0.1,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{meanFraction != null && meanFraction >= 0 && meanFraction <= 1 && (
|
||||
<div
|
||||
className="absolute bottom-0 top-0 w-px border-l border-dashed border-warm-400 dark:border-warm-500"
|
||||
style={{ left: `${meanFraction * 100}%` }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SkeletonHistogram() {
|
||||
return (
|
||||
<div className="bg-warm-50 dark:bg-warm-800 rounded p-2 animate-pulse">
|
||||
<div className="flex justify-between items-baseline">
|
||||
<div className="h-3 w-24 bg-warm-200 dark:bg-warm-700 rounded" />
|
||||
<div className="h-3 w-10 bg-warm-200 dark:bg-warm-700 rounded" />
|
||||
</div>
|
||||
<div className="flex items-end gap-px h-10 mt-2">
|
||||
{Array.from({ length: 15 }).map((_, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="flex-1 bg-warm-200 dark:bg-warm-700 rounded-t-sm min-w-[2px]"
|
||||
style={{ height: `${20 + Math.sin(i * 0.7) * 30 + 30}%` }}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex justify-between mt-1">
|
||||
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
|
||||
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function LoadingSkeleton() {
|
||||
return (
|
||||
<div className="p-3 space-y-4">
|
||||
{[0, 1, 2].map((groupIdx) => (
|
||||
<div key={groupIdx}>
|
||||
<div className="h-3 w-20 bg-warm-200 dark:bg-warm-700 rounded animate-pulse mb-2" />
|
||||
<div className="space-y-3">
|
||||
{Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => (
|
||||
<SkeletonHistogram key={i} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Map app property types to each site's expected values
|
||||
const PROPERTY_TYPE_MAP: Record<string, { rightmove: string; onthemarket: string; zoopla: string }> = {
|
||||
'House': { rightmove: 'detached,semi-detached,terraced', onthemarket: 'property', zoopla: '' },
|
||||
'Detached': { rightmove: 'detached', onthemarket: 'detached', zoopla: 'detached' },
|
||||
'Semi-Detached': { rightmove: 'semi-detached', onthemarket: 'semi-detached', zoopla: 'semi_detached' },
|
||||
'Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||
'End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||
'Enclosed Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||
'Enclosed End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||
'Flat': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' },
|
||||
'Maisonette': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' },
|
||||
'Bungalow': { rightmove: 'bungalow', onthemarket: 'bungalow', zoopla: 'bungalow' },
|
||||
'Park home': { rightmove: 'park-home', onthemarket: 'property', zoopla: '' },
|
||||
};
|
||||
|
||||
// Approximate H3 hex edge length in miles by resolution
|
||||
// See https://h3geo.org/docs/core-library/restable
|
||||
const H3_RADIUS_MILES: Record<number, number> = {
|
||||
4: 15, // ~24km edge → ~15mi
|
||||
5: 6, // ~9km → ~6mi
|
||||
6: 3, // ~3.5km → ~3mi
|
||||
7: 1, // ~1.3km → ~1mi
|
||||
8: 0.5, // ~0.5km → ~0.3mi, round up
|
||||
9: 0.25, // ~0.17km
|
||||
10: 0.25, // ~0.07km
|
||||
11: 0.25, // ~0.025km
|
||||
12: 0.25,
|
||||
};
|
||||
|
||||
// Rightmove only accepts specific radius values
|
||||
const RIGHTMOVE_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40];
|
||||
// OnTheMarket and Zoopla accept similar sets
|
||||
const OTM_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 30, 40];
|
||||
const ZOOPLA_RADII = [0.25, 0.5, 1, 3, 5, 10, 15, 20, 25, 30];
|
||||
|
||||
function nearestRadius(target: number, allowed: number[]): number {
|
||||
return allowed.reduce((best, r) => (Math.abs(r - target) < Math.abs(best - target) ? r : best));
|
||||
}
|
||||
|
||||
function buildPropertySearchUrls(
|
||||
location: HexagonLocation,
|
||||
filters: FeatureFilters
|
||||
): { rightmove: string; onthemarket: string; zoopla: string } {
|
||||
const { lat, lon, postcode, resolution } = location;
|
||||
const radiusMiles = H3_RADIUS_MILES[resolution] ?? 1;
|
||||
const coordStr = `${lat.toFixed(5)},${lon.toFixed(5)}`;
|
||||
|
||||
// Extract price filters
|
||||
const priceFilter = filters['Last known price'];
|
||||
const minPrice = Array.isArray(priceFilter) && typeof priceFilter[0] === 'number' ? priceFilter[0] : undefined;
|
||||
const maxPrice = Array.isArray(priceFilter) && typeof priceFilter[1] === 'number' ? priceFilter[1] : undefined;
|
||||
|
||||
// Extract property type filters
|
||||
const propertyTypes = filters['Property type'];
|
||||
const selectedTypes = Array.isArray(propertyTypes) && typeof propertyTypes[0] === 'string' ? propertyTypes as string[] : [];
|
||||
|
||||
// --- Rightmove ---
|
||||
// Rightmove accepts both postcodes and lat,lon in searchLocation
|
||||
const rmParams = new URLSearchParams();
|
||||
rmParams.set('searchLocation', postcode || coordStr);
|
||||
rmParams.set('channel', 'BUY');
|
||||
rmParams.set('radius', String(nearestRadius(radiusMiles, RIGHTMOVE_RADII)));
|
||||
if (minPrice !== undefined) rmParams.set('minPrice', String(Math.round(minPrice)));
|
||||
if (maxPrice !== undefined) rmParams.set('maxPrice', String(Math.round(maxPrice)));
|
||||
if (selectedTypes.length > 0) {
|
||||
const rmTypes = [...new Set(selectedTypes.flatMap((t) => {
|
||||
const mapped = PROPERTY_TYPE_MAP[t]?.rightmove;
|
||||
return mapped ? mapped.split(',') : [];
|
||||
}))];
|
||||
if (rmTypes.length > 0) rmParams.set('propertyTypes', rmTypes.join(','));
|
||||
}
|
||||
const rightmove = `https://www.rightmove.co.uk/property-for-sale/find.html?${rmParams.toString()}`;
|
||||
|
||||
// --- OnTheMarket ---
|
||||
let otmType = 'property';
|
||||
if (selectedTypes.length > 0) {
|
||||
const otmTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.onthemarket).filter(Boolean))];
|
||||
if (otmTypes.length === 1 && otmTypes[0] !== 'property') otmType = otmTypes[0]!;
|
||||
}
|
||||
const otmParams = new URLSearchParams();
|
||||
otmParams.set('radius', String(nearestRadius(radiusMiles, OTM_RADII)));
|
||||
if (minPrice !== undefined) otmParams.set('min-price', String(Math.round(minPrice)));
|
||||
if (maxPrice !== undefined) otmParams.set('max-price', String(Math.round(maxPrice)));
|
||||
let onthemarket: string;
|
||||
if (postcode) {
|
||||
const slug = postcode.replace(/\s+/g, '-').toLowerCase();
|
||||
onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/${slug}/?${otmParams.toString()}`;
|
||||
} else {
|
||||
// Use lat/lon search with geo params for bigger hexagons without a postcode
|
||||
otmParams.set('search-site', 'geo');
|
||||
otmParams.set('geo-lat', String(lat));
|
||||
otmParams.set('geo-lng', String(lon));
|
||||
onthemarket = `https://www.onthemarket.com/for-sale/${otmType}/?${otmParams.toString()}`;
|
||||
}
|
||||
|
||||
// --- Zoopla ---
|
||||
const zParams = new URLSearchParams();
|
||||
zParams.set('q', postcode || coordStr);
|
||||
zParams.set('search_source', 'for-sale');
|
||||
zParams.set('radius', String(nearestRadius(radiusMiles, ZOOPLA_RADII)));
|
||||
if (minPrice !== undefined) zParams.set('price_min', String(Math.round(minPrice)));
|
||||
if (maxPrice !== undefined) zParams.set('price_max', String(Math.round(maxPrice)));
|
||||
if (selectedTypes.length > 0) {
|
||||
const zTypes = [...new Set(selectedTypes.map((t) => PROPERTY_TYPE_MAP[t]?.zoopla).filter(Boolean))];
|
||||
for (const zt of zTypes) {
|
||||
zParams.append('property_sub_type', zt!);
|
||||
}
|
||||
}
|
||||
let zoopla: string;
|
||||
if (postcode) {
|
||||
const slug = postcode.replace(/\s+/g, '-').toLowerCase();
|
||||
zoopla = `https://www.zoopla.co.uk/for-sale/property/${slug}/?${zParams.toString()}`;
|
||||
} else {
|
||||
// Use coordinate-based path for bigger hexagons
|
||||
zParams.set('geo_autocomplete_identifier', `geo_${lat}_${lon}`);
|
||||
zoopla = `https://www.zoopla.co.uk/for-sale/property/?${zParams.toString()}`;
|
||||
}
|
||||
|
||||
return { rightmove, onthemarket, zoopla };
|
||||
}
|
||||
|
||||
function ExternalSearchLinks({ location, filters }: { location: HexagonLocation; filters: FeatureFilters }) {
|
||||
const urls = useMemo(() => buildPropertySearchUrls(location, filters), [location, filters]);
|
||||
const radiusMiles = H3_RADIUS_MILES[location.resolution] ?? 1;
|
||||
const label = location.postcode || `${radiusMiles}mi radius`;
|
||||
|
||||
return (
|
||||
<div className="p-3 border-b border-warm-200 dark:border-navy-700">
|
||||
<h3 className="text-xs font-semibold text-warm-500 dark:text-warm-400 uppercase tracking-wider mb-2">
|
||||
Search {label} on
|
||||
</h3>
|
||||
<div className="flex gap-2">
|
||||
<a
|
||||
href={urls.rightmove}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex-1 text-center text-xs py-1.5 px-2 rounded border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-teal-600 dark:text-teal-400 hover:bg-warm-50 dark:hover:bg-warm-700 font-medium"
|
||||
>
|
||||
Rightmove
|
||||
</a>
|
||||
<a
|
||||
href={urls.onthemarket}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex-1 text-center text-xs py-1.5 px-2 rounded border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-teal-600 dark:text-teal-400 hover:bg-warm-50 dark:hover:bg-warm-700 font-medium"
|
||||
>
|
||||
OnTheMarket
|
||||
</a>
|
||||
<a
|
||||
href={urls.zoopla}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex-1 text-center text-xs py-1.5 px-2 rounded border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-teal-600 dark:text-teal-400 hover:bg-warm-50 dark:hover:bg-warm-700 font-medium"
|
||||
>
|
||||
Zoopla
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function EnumBarChart({ counts }: { counts: Record<string, number> }) {
|
||||
const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA);
|
||||
const maxCount = Math.max(...entries.map(([, count]) => count), 1);
|
||||
|
|
@ -99,6 +352,8 @@ export default function AreaPane({
|
|||
onHoverModeChange,
|
||||
onViewProperties,
|
||||
onClose,
|
||||
hexagonLocation,
|
||||
filters,
|
||||
}: AreaPaneProps) {
|
||||
const featureGroups = useMemo(() => groupFeatures(globalFeatures), [globalFeatures]);
|
||||
|
||||
|
|
@ -113,6 +368,12 @@ export default function AreaPane({
|
|||
return new Map(stats.enum_features.map((feature) => [feature.name, feature]));
|
||||
}, [stats]);
|
||||
|
||||
// Build lookup for global feature metadata (for histogram overlay)
|
||||
const globalFeatureByName = useMemo(
|
||||
() => new Map(globalFeatures.map((f) => [f.name, f])),
|
||||
[globalFeatures]
|
||||
);
|
||||
|
||||
if (!hexagonId) {
|
||||
return (
|
||||
<div className="flex items-center justify-center h-full text-warm-500 dark:text-warm-400 px-4 text-center text-sm">
|
||||
|
|
@ -174,10 +435,13 @@ export default function AreaPane({
|
|||
)}
|
||||
</div>
|
||||
|
||||
{/* External search links */}
|
||||
{hexagonLocation && stats && <ExternalSearchLinks location={hexagonLocation} filters={filters} />}
|
||||
|
||||
{/* Stats content */}
|
||||
<div className="flex-1 overflow-y-auto">
|
||||
{loading && !stats ? (
|
||||
<div className="p-4 text-warm-500 dark:text-warm-400 text-sm">Loading...</div>
|
||||
<LoadingSkeleton />
|
||||
) : stats ? (
|
||||
<div className="p-3 space-y-4">
|
||||
{featureGroups.map((group) => {
|
||||
|
|
@ -198,9 +462,24 @@ export default function AreaPane({
|
|||
const enumStats = enumByName.get(feature.name);
|
||||
|
||||
if (numericStats) {
|
||||
const maxCount = Math.max(...numericStats.histogram.counts);
|
||||
const globalFeature = globalFeatureByName.get(feature.name);
|
||||
const globalHistogram = globalFeature?.histogram;
|
||||
// Compute a global mean from the global histogram for the mean line
|
||||
let globalMean: number | undefined;
|
||||
if (globalHistogram && globalHistogram.counts.length > 0) {
|
||||
const totalCount = globalHistogram.counts.reduce((a, b) => a + b, 0);
|
||||
if (totalCount > 0) {
|
||||
let weightedSum = 0;
|
||||
for (let i = 0; i < globalHistogram.counts.length; i++) {
|
||||
const binCenter = globalHistogram.min + (i + 0.5) * globalHistogram.bin_width;
|
||||
weightedSum += binCenter * globalHistogram.counts[i];
|
||||
}
|
||||
globalMean = weightedSum / totalCount;
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div key={feature.name} className="bg-warm-50 dark:bg-navy-800 rounded p-2">
|
||||
<div key={feature.name} className="bg-warm-50 dark:bg-warm-800 rounded p-2">
|
||||
<div className="flex justify-between items-baseline">
|
||||
<span className="text-xs text-warm-700 dark:text-warm-300 truncate mr-2">
|
||||
{feature.name}
|
||||
|
|
@ -210,17 +489,32 @@ export default function AreaPane({
|
|||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between text-[10px] text-warm-400 dark:text-warm-500 mt-0.5">
|
||||
<span>{formatValue(numericStats.min)}</span>
|
||||
<span>{formatValue(numericStats.max)}</span>
|
||||
<span>{formatValue(numericStats.histogram.min)}</span>
|
||||
<span>{formatValue(numericStats.histogram.max)}</span>
|
||||
</div>
|
||||
<MiniHistogram counts={numericStats.histogram.counts} maxCount={maxCount} />
|
||||
{globalHistogram ? (
|
||||
<DualHistogram
|
||||
localCounts={numericStats.histogram.counts}
|
||||
globalCounts={globalHistogram.counts}
|
||||
min={numericStats.histogram.min}
|
||||
max={numericStats.histogram.max}
|
||||
globalMean={globalMean}
|
||||
/>
|
||||
) : (
|
||||
<DualHistogram
|
||||
localCounts={numericStats.histogram.counts}
|
||||
globalCounts={numericStats.histogram.counts}
|
||||
min={numericStats.histogram.min}
|
||||
max={numericStats.histogram.max}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (enumStats) {
|
||||
return (
|
||||
<div key={feature.name} className="bg-warm-50 dark:bg-navy-800 rounded p-2">
|
||||
<div key={feature.name} className="bg-warm-50 dark:bg-warm-800 rounded p-2">
|
||||
<span className="text-xs text-warm-700 dark:text-warm-300">
|
||||
{feature.name}
|
||||
</span>
|
||||
|
|
|
|||
|
|
@ -654,6 +654,19 @@ export default memo(function Map({
|
|||
<DeckOverlay layers={layers} getTooltip={null} />
|
||||
</MapGL>
|
||||
<PostcodeSearch onFlyTo={handleFlyTo} />
|
||||
{viewSource === 'eye' && viewFeature && (
|
||||
<div className="absolute top-4 left-1/2 -translate-x-1/2 z-20 flex items-center gap-3 bg-white dark:bg-warm-800 rounded-lg shadow-lg px-5 py-3">
|
||||
<span className="text-lg font-semibold text-navy-950 dark:text-warm-100">
|
||||
Previewing “{viewFeature}”
|
||||
</span>
|
||||
<button
|
||||
onClick={onCancelPin}
|
||||
className="px-4 py-1.5 rounded-md bg-warm-200 dark:bg-warm-700 text-warm-700 dark:text-warm-200 hover:bg-warm-300 dark:hover:bg-warm-600 font-medium text-sm"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{viewFeature && colorRange && colorFeatureMeta ? (
|
||||
<MapLegend
|
||||
featureLabel={colorFeatureMeta.name}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import { createRoot } from 'react-dom/client';
|
||||
import App from './App';
|
||||
import './index.css';
|
||||
import { initPlausible } from './usePlausible';
|
||||
|
||||
initPlausible();
|
||||
|
||||
const container = document.getElementById('root');
|
||||
if (!container) {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ export interface FeatureMeta {
|
|||
min?: number;
|
||||
max?: number;
|
||||
step?: number;
|
||||
histogram?: { min: number; max: number; bin_width: number; counts: number[] };
|
||||
// Enum-only fields
|
||||
values?: string[];
|
||||
// Description fields
|
||||
|
|
|
|||
73
frontend/src/usePlausible.ts
Normal file
73
frontend/src/usePlausible.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
const DOMAIN = 'narrowit.schmelczer.dev';
|
||||
const ENDPOINT = '/status';
|
||||
|
||||
type EventOptions = {
|
||||
props?: Record<string, string | number | boolean>;
|
||||
revenue?: { currency: string; amount: number };
|
||||
};
|
||||
|
||||
function sendEvent(name: string, options?: EventOptions) {
|
||||
const payload: Record<string, unknown> = {
|
||||
n: name,
|
||||
u: window.location.href,
|
||||
d: DOMAIN,
|
||||
r: document.referrer || null,
|
||||
};
|
||||
if (options?.props) {
|
||||
payload.p = JSON.stringify(options.props);
|
||||
}
|
||||
if (options?.revenue) {
|
||||
payload.$ = JSON.stringify(options.revenue);
|
||||
}
|
||||
if (navigator.sendBeacon) {
|
||||
navigator.sendBeacon(
|
||||
ENDPOINT,
|
||||
new Blob([JSON.stringify(payload)], { type: 'application/json' })
|
||||
);
|
||||
} else {
|
||||
fetch(ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
keepalive: true,
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
let initialized = false;
|
||||
|
||||
/**
|
||||
* Tracks pageview on first call and returns a trackEvent function.
|
||||
* Tracks outbound link clicks automatically.
|
||||
*/
|
||||
export function initPlausible() {
|
||||
if (initialized) return;
|
||||
initialized = true;
|
||||
|
||||
// Initial pageview
|
||||
sendEvent('pageview');
|
||||
|
||||
// Track outbound link clicks
|
||||
document.addEventListener('click', (e) => {
|
||||
const link = (e.target as HTMLElement).closest?.('a');
|
||||
if (!link) return;
|
||||
const href = link.getAttribute('href');
|
||||
if (!href) return;
|
||||
try {
|
||||
const url = new URL(href, window.location.origin);
|
||||
if (url.hostname !== window.location.hostname) {
|
||||
sendEvent('Outbound Link: Click', { props: { url: href } });
|
||||
}
|
||||
} catch {
|
||||
// invalid URL, ignore
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function trackPageview(options?: EventOptions) {
|
||||
sendEvent('pageview', options);
|
||||
}
|
||||
|
||||
export function trackEvent(name: string, options?: EventOptions) {
|
||||
sendEvent(name, options);
|
||||
}
|
||||
|
|
@ -51,6 +51,12 @@ module.exports = (env, argv) => {
|
|||
context: ['/api'],
|
||||
target: 'http://localhost:8001',
|
||||
},
|
||||
{
|
||||
context: ['/status'],
|
||||
target: 'https://stats.schmelczer.dev',
|
||||
changeOrigin: true,
|
||||
pathRewrite: { '^/status': '/api/event' },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
|
|
|||
10
server-rs/Cargo.lock
generated
10
server-rs/Cargo.lock
generated
|
|
@ -1031,6 +1031,15 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lasso"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb"
|
||||
dependencies = [
|
||||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
|
|
@ -1813,6 +1822,7 @@ dependencies = [
|
|||
"axum",
|
||||
"clap",
|
||||
"h3o",
|
||||
"lasso",
|
||||
"polars",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ h3o = "0.7"
|
|||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
rayon = "1"
|
||||
lasso = "0.7"
|
||||
rustc-hash = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
pub const HISTOGRAM_BINS: usize = 100;
|
||||
|
||||
pub const H3_PRECOMPUTE_MIN: u8 = 4;
|
||||
pub const H3_PRECOMPUTE_MIN: u8 = 7;
|
||||
pub const H3_PRECOMPUTE_MAX: u8 = 12;
|
||||
pub const H3_REQUEST_MIN: u8 = 4;
|
||||
pub const H3_REQUEST_MAX: u8 = 12;
|
||||
|
||||
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
|
||||
|
||||
pub const BOUNDS_QUANTIZATION: f64 = 0.01;
|
||||
pub const BOUNDS_BUFFER_PERCENT: f64 = 0.1;
|
||||
pub const GRID_CELL_SIZE: f32 = 0.01;
|
||||
pub const POSTCODE_MIN_RESOLUTION: u8 = 11;
|
||||
pub const MAX_POIS_PER_REQUEST: usize = 2500;
|
||||
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
pub enum Bounds {
|
||||
/// Fixed min/max values for the slider
|
||||
Fixed { min: f64, max: f64 },
|
||||
Fixed { min: f32, max: f32 },
|
||||
/// Compute percentile from data at startup
|
||||
Percentile { low: f64, high: f64 },
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ pub struct FeatureConfig {
|
|||
pub name: &'static str,
|
||||
pub bounds: Bounds,
|
||||
/// Slider step size. Controls the granularity of the range slider in the UI.
|
||||
pub step: f64,
|
||||
pub step: f32,
|
||||
/// Short one-line description shown in the filter sidebar
|
||||
pub description: &'static str,
|
||||
/// Longer description explaining methodology, data source, and caveats
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ use crate::data::EnumFeatureData;
|
|||
|
||||
pub struct ParsedFilter {
|
||||
pub feat_idx: usize,
|
||||
pub min: f64,
|
||||
pub max: f64,
|
||||
pub min: f32,
|
||||
pub max: f32,
|
||||
}
|
||||
|
||||
pub struct ParsedEnumFilter {
|
||||
|
|
@ -51,11 +51,11 @@ pub fn parse_filters(
|
|||
if num_parts.len() != 2 {
|
||||
continue;
|
||||
}
|
||||
let min = match num_parts[0].trim().parse::<f64>() {
|
||||
let min = match num_parts[0].trim().parse::<f32>() {
|
||||
Ok(value) => value,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let max = match num_parts[1].trim().parse::<f64>() {
|
||||
let max = match num_parts[1].trim().parse::<f32>() {
|
||||
Ok(value) => value,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
|
@ -72,7 +72,7 @@ pub fn row_passes_filters(
|
|||
row: usize,
|
||||
filters: &[ParsedFilter],
|
||||
enum_filters: &[ParsedEnumFilter],
|
||||
feature_data: &[f64],
|
||||
feature_data: &[f32],
|
||||
num_features: usize,
|
||||
enum_features: &[EnumFeatureData],
|
||||
) -> bool {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
/// Divides the UK bounding box into cells of ~0.01 degrees (~1km),
|
||||
/// each storing indices of rows whose lat/lon falls within that cell.
|
||||
pub struct GridIndex {
|
||||
min_lat: f64,
|
||||
min_lon: f64,
|
||||
cell_size: f64,
|
||||
min_lat: f32,
|
||||
min_lon: f32,
|
||||
cell_size: f32,
|
||||
cols: usize,
|
||||
rows: usize,
|
||||
/// cells[row * cols + col] = vec of row indices
|
||||
|
|
@ -13,11 +13,11 @@ pub struct GridIndex {
|
|||
}
|
||||
|
||||
impl GridIndex {
|
||||
pub fn build(lat: &[f64], lon: &[f64], cell_size: f64) -> Self {
|
||||
let mut min_lat = f64::INFINITY;
|
||||
let mut max_lat = f64::NEG_INFINITY;
|
||||
let mut min_lon = f64::INFINITY;
|
||||
let mut max_lon = f64::NEG_INFINITY;
|
||||
pub fn build(lat: &[f32], lon: &[f32], cell_size: f32) -> Self {
|
||||
let mut min_lat = f32::INFINITY;
|
||||
let mut max_lat = f32::NEG_INFINITY;
|
||||
let mut min_lon = f32::INFINITY;
|
||||
let mut max_lon = f32::NEG_INFINITY;
|
||||
|
||||
for index in 0..lat.len() {
|
||||
if lat[index] < min_lat {
|
||||
|
|
@ -71,6 +71,7 @@ impl GridIndex {
|
|||
}
|
||||
}
|
||||
|
||||
/// Query accepts f64 bounds (from HTTP parsing) and casts internally.
|
||||
pub fn query(&self, south: f64, west: f64, north: f64, east: f64) -> Vec<u32> {
|
||||
let Some((row_min, row_max, col_min, col_max)) =
|
||||
self.clamp_bounds(south, west, north, east)
|
||||
|
|
@ -121,10 +122,14 @@ impl GridIndex {
|
|||
north: f64,
|
||||
east: f64,
|
||||
) -> Option<(usize, usize, usize, usize)> {
|
||||
let row_min_raw = ((south - self.min_lat) / self.cell_size) as isize;
|
||||
let row_max_raw = ((north - self.min_lat) / self.cell_size) as isize;
|
||||
let col_min_raw = ((west - self.min_lon) / self.cell_size) as isize;
|
||||
let col_max_raw = ((east - self.min_lon) / self.cell_size) as isize;
|
||||
let min_lat = self.min_lat as f64;
|
||||
let min_lon = self.min_lon as f64;
|
||||
let cell_size = self.cell_size as f64;
|
||||
|
||||
let row_min_raw = ((south - min_lat) / cell_size) as isize;
|
||||
let row_max_raw = ((north - min_lat) / cell_size) as isize;
|
||||
let col_min_raw = ((west - min_lon) / cell_size) as isize;
|
||||
let col_max_raw = ((east - min_lon) / cell_size) as isize;
|
||||
|
||||
let row_min = row_min_raw.max(0) as usize;
|
||||
let row_max_clamped = row_max_raw.min(self.rows as isize - 1);
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
);
|
||||
|
||||
info!("Building spatial grid index (0.01° cells)");
|
||||
let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, 0.01);
|
||||
let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, consts::GRID_CELL_SIZE);
|
||||
|
||||
info!(
|
||||
"Precomputing H3 cells for resolutions {}-{}",
|
||||
|
|
@ -89,7 +89,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
info!(pois = poi_data.lat.len(), "POI data loaded");
|
||||
|
||||
info!("Building POI spatial grid index");
|
||||
let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, 0.01);
|
||||
let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);
|
||||
|
||||
let min_keys: Vec<String> = property_data
|
||||
.feature_names
|
||||
|
|
@ -116,11 +116,14 @@ async fn main() -> anyhow::Result<()> {
|
|||
let poi_category_groups = {
|
||||
let mut group_cats: std::collections::HashMap<String, std::collections::HashSet<String>> =
|
||||
std::collections::HashMap::new();
|
||||
for (category, group) in poi_data.category.iter().zip(poi_data.group.iter()) {
|
||||
let num_pois = poi_data.category.indices.len();
|
||||
for row in 0..num_pois {
|
||||
let category = poi_data.category.get(row).to_string();
|
||||
let group = poi_data.group.get(row).to_string();
|
||||
group_cats
|
||||
.entry(group.clone())
|
||||
.entry(group)
|
||||
.or_default()
|
||||
.insert(category.clone());
|
||||
.insert(category);
|
||||
}
|
||||
// Validate that data groups match the hardcoded order exactly
|
||||
let expected: std::collections::HashSet<&str> =
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ pub enum FeatureInfo {
|
|||
#[serde(rename = "numeric")]
|
||||
Numeric {
|
||||
name: String,
|
||||
min: f64,
|
||||
max: f64,
|
||||
step: f64,
|
||||
min: f32,
|
||||
max: f32,
|
||||
step: f32,
|
||||
histogram: Histogram,
|
||||
description: &'static str,
|
||||
detail: &'static str,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use axum::response::IntoResponse;
|
|||
use serde::Deserialize;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{ENUM_NULL, HISTOGRAM_BINS};
|
||||
use crate::consts::{ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, HISTOGRAM_BINS};
|
||||
use crate::filter::{parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
||||
|
|
@ -31,17 +31,21 @@ pub async fn get_hexagon_stats(
|
|||
})?;
|
||||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution as usize;
|
||||
if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() {
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Invalid or non-precomputed resolution for hexagon-stats"
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid or non-precomputed resolution".to_string(),
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
let resolution_idx = resolution as usize;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
|
|
@ -54,7 +58,13 @@ pub async fn get_hexagon_stats(
|
|||
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let start_time = std::time::Instant::now();
|
||||
let h3_data = &state.h3_cells[resolution];
|
||||
let precomputed: Option<&[u64]> = state
|
||||
.h3_cells
|
||||
.get(resolution_idx)
|
||||
.filter(|cells| !cells.is_empty())
|
||||
.map(|cells| cells.as_slice());
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let enum_features = &state.data.enum_features;
|
||||
|
|
@ -67,7 +77,14 @@ pub async fn get_hexagon_stats(
|
|||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if h3_data[row] == cell_u64
|
||||
let row_cell = if let Some(h3_data) = precomputed {
|
||||
h3_data[row]
|
||||
} else {
|
||||
h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if row_cell == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -98,9 +115,9 @@ pub async fn get_hexagon_stats(
|
|||
let bin_width = global_stats.histogram.bin_width;
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f64::INFINITY;
|
||||
let mut max_value = f64::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64; // keep f64 for mean precision
|
||||
let mut bins = vec![0u64; HISTOGRAM_BINS];
|
||||
|
||||
for &row in &matching_rows {
|
||||
|
|
@ -113,12 +130,12 @@ pub async fn get_hexagon_stats(
|
|||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value;
|
||||
sum += value as f64;
|
||||
|
||||
// Bin into histogram using global edges
|
||||
// Bin into histogram using global edges (cast to f64 for bin index math)
|
||||
if bin_width > 0.0 {
|
||||
let bin_index =
|
||||
((value - histogram_min) / bin_width).floor() as isize;
|
||||
((value as f64 - histogram_min as f64) / bin_width as f64).floor() as isize;
|
||||
let clamped_index = bin_index.max(0).min((HISTOGRAM_BINS - 1) as isize) as usize;
|
||||
bins[clamped_index] += 1;
|
||||
}
|
||||
|
|
@ -138,15 +155,15 @@ pub async fn get_hexagon_stats(
|
|||
output.push_str("{\"name\":");
|
||||
write_json_string(&mut output, feature_name);
|
||||
write!(output, ",\"count\":{}", count).unwrap();
|
||||
write!(output, ",\"min\":{}", format_f64(min_value)).unwrap();
|
||||
write!(output, ",\"max\":{}", format_f64(max_value)).unwrap();
|
||||
write!(output, ",\"min\":{}", format_num(min_value)).unwrap();
|
||||
write!(output, ",\"max\":{}", format_num(max_value)).unwrap();
|
||||
write!(output, ",\"mean\":{}", format_f64(mean)).unwrap();
|
||||
output.push_str(",\"histogram\":{\"min\":");
|
||||
write!(output, "{}", format_f64(histogram_min)).unwrap();
|
||||
write!(output, "{}", format_num(histogram_min)).unwrap();
|
||||
output.push_str(",\"max\":");
|
||||
write!(output, "{}", format_f64(histogram_max)).unwrap();
|
||||
write!(output, "{}", format_num(histogram_max)).unwrap();
|
||||
output.push_str(",\"bin_width\":");
|
||||
write!(output, "{}", format_f64(bin_width)).unwrap();
|
||||
write!(output, "{}", format_num(bin_width)).unwrap();
|
||||
output.push_str(",\"counts\":[");
|
||||
for (bin_index, &bin_count) in bins.iter().enumerate() {
|
||||
if bin_index > 0 {
|
||||
|
|
@ -216,10 +233,11 @@ pub async fn get_hexagon_stats(
|
|||
"GET /api/hexagon-stats"
|
||||
);
|
||||
|
||||
output
|
||||
Ok(output)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?;
|
||||
|
||||
Ok((
|
||||
[(axum::http::header::CONTENT_TYPE, "application/json")],
|
||||
|
|
@ -242,6 +260,15 @@ fn write_json_string(output: &mut String, value: &str) {
|
|||
output.push('"');
|
||||
}
|
||||
|
||||
fn format_num(value: f32) -> String {
|
||||
let fv = value as f64;
|
||||
if fv.fract() == 0.0 && fv.abs() < 1e15 {
|
||||
format!("{:.1}", fv)
|
||||
} else {
|
||||
format!("{}", fv)
|
||||
}
|
||||
}
|
||||
|
||||
fn format_f64(value: f64) -> String {
|
||||
if value.fract() == 0.0 && value.abs() < 1e15 {
|
||||
format!("{:.1}", value)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use serde::Deserialize;
|
|||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{
|
||||
BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_PRECOMPUTE_MIN,
|
||||
BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN,
|
||||
POSTCODE_MIN_RESOLUTION,
|
||||
};
|
||||
use crate::filter::parse_filters;
|
||||
|
|
@ -44,8 +44,8 @@ pub struct HexagonParams {
|
|||
/// Per-cell accumulator for aggregating features
|
||||
struct CellAgg {
|
||||
count: u32,
|
||||
mins: Vec<f64>,
|
||||
maxs: Vec<f64>,
|
||||
mins: Vec<f32>,
|
||||
maxs: Vec<f32>,
|
||||
/// Min/max ordinal indices for enum features (255 = no data yet)
|
||||
enum_mins: Vec<u8>,
|
||||
enum_maxs: Vec<u8>,
|
||||
|
|
@ -60,8 +60,8 @@ impl CellAgg {
|
|||
fn new(num_features: usize, num_enums: usize) -> Self {
|
||||
CellAgg {
|
||||
count: 0,
|
||||
mins: vec![f64::INFINITY; num_features],
|
||||
maxs: vec![f64::NEG_INFINITY; num_features],
|
||||
mins: vec![f32::INFINITY; num_features],
|
||||
maxs: vec![f32::NEG_INFINITY; num_features],
|
||||
enum_mins: vec![ENUM_NULL; num_enums],
|
||||
enum_maxs: vec![0; num_enums],
|
||||
postcode: None,
|
||||
|
|
@ -75,7 +75,7 @@ impl CellAgg {
|
|||
/// feature_data[row * num_features + feat_idx] — all features for one row
|
||||
/// are contiguous, so this reads a single cache line per ~8 features.
|
||||
#[inline]
|
||||
fn add_row(&mut self, feature_data: &[f64], row: usize, num_features: usize) {
|
||||
fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
|
|
@ -110,9 +110,9 @@ impl CellAgg {
|
|||
/// Track postcode and centroid for high-resolution cells.
|
||||
/// Uses simple "first seen" approach — at res 11/12, most rows in a cell share a postcode.
|
||||
#[inline]
|
||||
fn add_postcode(&mut self, postcode: &str, lat: f64, lon: f64) {
|
||||
self.lat_sum += lat;
|
||||
self.lon_sum += lon;
|
||||
fn add_postcode(&mut self, postcode: &str, lat: f32, lon: f32) {
|
||||
self.lat_sum += lat as f64;
|
||||
self.lon_sum += lon as f64;
|
||||
if postcode.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
|
@ -212,16 +212,16 @@ pub async fn get_hexagons(
|
|||
Query(params): Query<HexagonParams>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let resolution = params.resolution;
|
||||
if resolution < H3_PRECOMPUTE_MIN || resolution > H3_PRECOMPUTE_MAX {
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
|
|
@ -304,7 +304,7 @@ pub async fn get_hexagons(
|
|||
aggregation.add_enums(enum_features, row);
|
||||
if include_postcode {
|
||||
aggregation.add_postcode(
|
||||
&state.data.postcode[row],
|
||||
state.data.postcode(row),
|
||||
state.data.lat[row],
|
||||
state.data.lon[row],
|
||||
);
|
||||
|
|
@ -320,7 +320,7 @@ pub async fn get_hexagons(
|
|||
if !row_passes(row) {
|
||||
return;
|
||||
}
|
||||
let cell_id = h3o::LatLng::new(state.data.lat[row], state.data.lon[row])
|
||||
let cell_id = h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0);
|
||||
let aggregation = groups
|
||||
|
|
@ -330,7 +330,7 @@ pub async fn get_hexagons(
|
|||
aggregation.add_enums(enum_features, row);
|
||||
if include_postcode {
|
||||
aggregation.add_postcode(
|
||||
&state.data.postcode[row],
|
||||
state.data.postcode(row),
|
||||
state.data.lat[row],
|
||||
state.data.lon[row],
|
||||
);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ pub async fn get_pois(
|
|||
.filter_map(|&row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if let Some(ref categories) = category_filter {
|
||||
if !categories.contains(&state.poi_data.category[row]) {
|
||||
if !categories.contains(state.poi_data.category.get(row)) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
|
@ -83,11 +83,11 @@ pub async fn get_pois(
|
|||
.map(|&row| POI {
|
||||
id: state.poi_data.id[row].clone(),
|
||||
name: state.poi_data.name[row].clone(),
|
||||
category: state.poi_data.category[row].clone(),
|
||||
group: state.poi_data.group[row].clone(),
|
||||
category: state.poi_data.category.get(row).to_string(),
|
||||
group: state.poi_data.group.get(row).to_string(),
|
||||
lat: state.poi_data.lat[row],
|
||||
lng: state.poi_data.lng[row],
|
||||
emoji: state.poi_data.emoji[row].clone(),
|
||||
emoji: state.poi_data.emoji.get(row).to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use rustc_hash::FxHashMap;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, MAX_PROPERTIES_LIMIT};
|
||||
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT};
|
||||
use crate::data::EnumFeatureData;
|
||||
use crate::filter::{parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
|
@ -36,13 +36,13 @@ pub struct Property {
|
|||
pub potential_energy_rating: Option<String>,
|
||||
|
||||
// Numeric fields
|
||||
pub lat: f64,
|
||||
pub lon: f64,
|
||||
pub lat: f32,
|
||||
pub lon: f32,
|
||||
|
||||
pub is_construction_date_approximate: Option<bool>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub features: FxHashMap<String, f64>,
|
||||
pub features: FxHashMap<String, f32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -93,17 +93,21 @@ pub async fn get_hexagon_properties(
|
|||
})?;
|
||||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution as usize;
|
||||
if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() {
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Invalid or non-precomputed resolution for hexagon-properties"
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid or non-precomputed resolution".to_string(),
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
let resolution_idx = resolution as usize;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
|
|
@ -116,7 +120,13 @@ pub async fn get_hexagon_properties(
|
|||
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let t0 = std::time::Instant::now();
|
||||
let h3_data = &state.h3_cells[resolution];
|
||||
let precomputed: Option<&[u64]> = state
|
||||
.h3_cells
|
||||
.get(resolution_idx)
|
||||
.filter(|cells| !cells.is_empty())
|
||||
.map(|cells| cells.as_slice());
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let enum_features = &state.data.enum_features;
|
||||
|
|
@ -128,7 +138,14 @@ pub async fn get_hexagon_properties(
|
|||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if h3_data[row] == cell_u64
|
||||
let row_cell = if let Some(h3_data) = precomputed {
|
||||
h3_data[row]
|
||||
} else {
|
||||
h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if row_cell == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -162,8 +179,8 @@ pub async fn get_hexagon_properties(
|
|||
}
|
||||
|
||||
Property {
|
||||
address: non_empty_string(&state.data.address[row]),
|
||||
postcode: non_empty_string(&state.data.postcode[row]),
|
||||
address: non_empty_string(state.data.address(row)),
|
||||
postcode: non_empty_string(state.data.postcode(row)),
|
||||
is_construction_date_approximate: Some(state.data.is_approx_build_date[row]),
|
||||
property_type: lookup_enum_value(
|
||||
enum_features,
|
||||
|
|
@ -215,16 +232,17 @@ pub async fn get_hexagon_properties(
|
|||
"GET /api/hexagon-properties"
|
||||
);
|
||||
|
||||
HexagonPropertiesResponse {
|
||||
Ok(HexagonPropertiesResponse {
|
||||
properties,
|
||||
total,
|
||||
limit,
|
||||
offset,
|
||||
truncated,
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
.map_err(|error: String| (StatusCode::INTERNAL_SERVER_ERROR, error))?;
|
||||
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_bounds_fully_below_grid_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(10.0, -10.0, 20.0, -5.0);
|
||||
|
|
@ -17,8 +17,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_bounds_fully_above_grid_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(80.0, 50.0, 90.0, 60.0);
|
||||
|
|
@ -30,8 +30,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_inverted_bounds_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
// south > north
|
||||
|
|
@ -44,8 +44,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn for_each_bounds_fully_outside_yields_nothing() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let mut count = 0;
|
||||
|
|
@ -60,8 +60,8 @@ mod grid_index_tests {
|
|||
fn query_with_large_cells_outside_returns_empty() {
|
||||
// Previously, out-of-bounds queries with large cell sizes would
|
||||
// scan cell (0,0) which could contain data. Now returns empty.
|
||||
let lat = vec![50.0];
|
||||
let lon = vec![0.0];
|
||||
let lat = vec![50.0_f32];
|
||||
let lon = vec![0.0_f32];
|
||||
let grid = GridIndex::build(&lat, &lon, 1.0);
|
||||
|
||||
let results = grid.query(0.0, -50.0, 10.0, -40.0);
|
||||
|
|
@ -73,8 +73,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_within_bounds_returns_correct_results() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(49.9, -0.1, 51.1, 1.1);
|
||||
|
|
@ -83,8 +83,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_partial_bounds_returns_subset() {
|
||||
let lat = vec![50.0, 51.0, 52.0];
|
||||
let lon = vec![0.0, 0.0, 0.0];
|
||||
let lat = vec![50.0_f32, 51.0, 52.0];
|
||||
let lon = vec![0.0_f32, 0.0, 0.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(49.9, -0.1, 50.1, 0.1);
|
||||
|
|
@ -100,7 +100,7 @@ mod filter_tests {
|
|||
#[test]
|
||||
fn nan_rows_fail_numeric_filter_even_with_infinite_range() {
|
||||
let feature_names = vec!["price".to_string()];
|
||||
let feature_data = vec![f64::NAN];
|
||||
let feature_data = vec![f32::NAN];
|
||||
let enum_features: Vec<EnumFeatureData> = vec![];
|
||||
|
||||
let (numeric, enums) =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue