Refactor and other improvements
This commit is contained in:
parent
04a78e7bfe
commit
6c90cf3c0f
47 changed files with 2705 additions and 1568 deletions
|
|
@ -302,7 +302,7 @@ Follow these conventions in all Rust code:
|
|||
```
|
||||
4. **JSON serialization**: Use `serde_json` with `#[derive(Serialize)]` structs, not manual string building
|
||||
5. **Precompute at startup**: For static/rarely-changing responses, compute once at startup and store in `AppState`
|
||||
6. **Unique placeholders**: When injecting content into HTML, use distinctive markers like `__NARROWIT_OG_TAGS__` that won't accidentally match other content
|
||||
6. **Unique placeholders**: When injecting content into HTML, use distinctive markers like `__PERFECT_POSTCODES_OG_TAGS__` that won't accidentally match other content
|
||||
|
||||
## Key Implementation Details
|
||||
|
||||
|
|
@ -317,7 +317,7 @@ Follow these conventions in all Rust code:
|
|||
- **Server-side AABB filtering**: Both `/api/hexagons` and `/api/postcodes` filter results by bounding-box intersection with query bounds. Hexagons use `h3_cell_bounds()` (h3o returns degrees, not radians). Postcodes compute polygon AABB from vertices. See `bounds_intersect()` in `parsing/bounds.rs`.
|
||||
- **GridIndex returns slightly more than requested**: The 0.01° grid cells mean properties up to ~1km outside the viewport may be returned. The AABB filter in the route handlers catches these extras.
|
||||
- **POI proximity**: Uses 0.05° grid (~5km cells) to reduce candidates before haversine distance check
|
||||
- **OG tag injection**: Uses `<meta name="x-og-placeholder" content="__NARROWIT_OG_TAGS__"/>` placeholder in HTML, replaced at runtime by middleware
|
||||
- **OG tag injection**: Uses `<meta name="x-og-placeholder" content="__PERFECT_POSTCODES_OG_TAGS__"/>` placeholder in HTML, replaced at runtime by middleware
|
||||
|
||||
## Rust Performance Patterns (server-rs)
|
||||
|
||||
|
|
|
|||
235
Makefile.data
Normal file
235
Makefile.data
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
# Data pipeline — download sources and build wide.parquet
|
||||
#
|
||||
# Usage:
|
||||
# make -f Makefile.data prepare # Build wide.parquet (+ all deps)
|
||||
# make -f Makefile.data tiles # Download UK map tiles
|
||||
# make -f Makefile.data download-pois # Download a single dataset
|
||||
# make -f Makefile.data help # List all targets
|
||||
#
|
||||
# Or include from the main Makefile and use targets directly.
|
||||
|
||||
SHELL := /bin/bash
|
||||
.DELETE_ON_ERROR:
|
||||
|
||||
DATA_DIR := /bulk/property-data
|
||||
|
||||
# ── Output files ──────────────────────────────────────────────────────────────
|
||||
|
||||
TILES := $(DATA_DIR)/uk.pmtiles
|
||||
ARCGIS := $(DATA_DIR)/arcgis_data.parquet
|
||||
PRICE_PAID := $(DATA_DIR)/price-paid-complete.parquet
|
||||
IOD := $(DATA_DIR)/IoD2025_Scores.parquet
|
||||
POIS_RAW := $(DATA_DIR)/uk_pois.parquet
|
||||
POIS_FILTERED := $(DATA_DIR)/filtered_uk_pois.parquet
|
||||
POI_PROXIMITY := $(DATA_DIR)/poi_proximity.parquet
|
||||
EPC_PP := $(DATA_DIR)/epc_pp.parquet
|
||||
WIDE := $(DATA_DIR)/wide.parquet
|
||||
EPC := $(DATA_DIR)/certificates.csv
|
||||
JT_BANK := $(DATA_DIR)/journey_times_bank.parquet
|
||||
JT_FITZROVIA := $(DATA_DIR)/journey_times_fitzrovia.parquet
|
||||
ETHNICITY := $(DATA_DIR)/ethnicity_by_la.parquet
|
||||
CRIME_DIR := $(DATA_DIR)/crime
|
||||
CRIME := $(DATA_DIR)/crime_by_lsoa.parquet
|
||||
NOISE := $(DATA_DIR)/road_noise.parquet
|
||||
OFSTED := $(DATA_DIR)/ofsted.parquet
|
||||
NAPTAN := $(DATA_DIR)/naptan.parquet
|
||||
BROADBAND := $(DATA_DIR)/broadband.parquet
|
||||
SCHOOL_PROX := $(DATA_DIR)/school_proximity.parquet
|
||||
GEOSURE_DIR := $(DATA_DIR)/geosure
|
||||
GEOSURE := $(DATA_DIR)/geosure.parquet
|
||||
INSPIRE_DIR := $(DATA_DIR)/inspire
|
||||
OA_BOUNDARIES := $(DATA_DIR)/oa_boundaries.gpkg
|
||||
UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet
|
||||
PC_BOUNDARIES := $(DATA_DIR)/new_postcode_boundaries
|
||||
|
||||
# Sentinel files for directory targets (Make doesn't track directories well)
|
||||
GEOSURE_STAMP := $(GEOSURE_DIR)/.done
|
||||
INSPIRE_STAMP := $(INSPIRE_DIR)/.done
|
||||
|
||||
MANUAL_DATA := $(DATA_DIR)/manual_data
|
||||
PMTILES_VERSION := 1.22.3
|
||||
PMTILES_BIN := $(DATA_DIR)/pmtiles
|
||||
|
||||
# ── Phony aliases ─────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: prepare tiles \
|
||||
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
||||
download-naptan download-pois download-ofsted download-broadband \
|
||||
download-postcodes download-geosure download-noise download-inspire \
|
||||
download-oa-boundaries download-uprn-lookup \
|
||||
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
||||
transform-school-proximity transform-geosure transform-postcode-boundaries \
|
||||
journey-times
|
||||
|
||||
prepare: $(WIDE)
|
||||
tiles: $(TILES)
|
||||
download-arcgis: $(ARCGIS)
|
||||
download-price-paid: $(PRICE_PAID)
|
||||
download-deprivation: $(IOD)
|
||||
download-ethnicity: $(ETHNICITY)
|
||||
download-naptan: $(NAPTAN)
|
||||
download-pois: $(POIS_RAW)
|
||||
download-ofsted: $(OFSTED)
|
||||
download-broadband: $(BROADBAND)
|
||||
download-postcodes: $(POSTCODES)
|
||||
download-geosure: $(GEOSURE_STAMP)
|
||||
download-noise: $(NOISE)
|
||||
download-inspire: $(INSPIRE_STAMP)
|
||||
download-oa-boundaries: $(OA_BOUNDARIES)
|
||||
download-uprn-lookup: $(UPRN_LOOKUP)
|
||||
transform-pois: $(POIS_FILTERED)
|
||||
transform-epc-pp: $(EPC_PP)
|
||||
transform-crime: $(CRIME)
|
||||
transform-poi-proximity: $(POI_PROXIMITY)
|
||||
transform-school-proximity: $(SCHOOL_PROX)
|
||||
transform-geosure: $(GEOSURE)
|
||||
transform-postcode-boundaries: $(PC_BOUNDARIES)
|
||||
|
||||
# ── Downloads ─────────────────────────────────────────────────────────────────
|
||||
|
||||
$(TILES):
|
||||
@echo "Downloading UK PMTiles (~1.5GB)..."
|
||||
@echo "This extracts UK tiles from the Protomaps planet file."
|
||||
@if [ ! -f "$(PMTILES_BIN)" ]; then \
|
||||
echo "Downloading pmtiles CLI v$(PMTILES_VERSION)..."; \
|
||||
curl -sL "https://github.com/protomaps/go-pmtiles/releases/download/v$(PMTILES_VERSION)/go-pmtiles_$(PMTILES_VERSION)_Linux_x86_64.tar.gz" \
|
||||
| tar -xz -C "$(DATA_DIR)" pmtiles; \
|
||||
chmod +x "$(PMTILES_BIN)"; \
|
||||
fi
|
||||
"$(PMTILES_BIN)" extract https://build.protomaps.com/20260201.pmtiles $@ --bbox=-10.5,49.5,2.5,61
|
||||
|
||||
# EPC requires manual registration — fail with instructions
|
||||
$(EPC):
|
||||
@echo ""
|
||||
@echo "=== EPC dataset not found ==="
|
||||
@echo "The EPC certificates file is required: $@"
|
||||
@echo ""
|
||||
@echo "To obtain it, register at https://epc.opendatacommunities.org/login"
|
||||
@echo ""
|
||||
@exit 1
|
||||
|
||||
$(ARCGIS):
|
||||
uv run python -m pipeline.download.arcgis --output $@
|
||||
|
||||
$(PRICE_PAID):
|
||||
uv run python -m pipeline.download.price_paid --output $@
|
||||
|
||||
$(IOD):
|
||||
uv run python -m pipeline.download.deprivation_data --output $@
|
||||
|
||||
$(ETHNICITY):
|
||||
uv run python -m pipeline.download.ethnicity --output $@
|
||||
|
||||
$(NAPTAN):
|
||||
uv run python -m pipeline.download.naptan --output $@
|
||||
|
||||
$(POIS_RAW):
|
||||
uv run python -m pipeline.download.pois --output $@
|
||||
|
||||
$(OFSTED):
|
||||
uv run python -m pipeline.download.ofsted --output $@
|
||||
|
||||
$(BROADBAND):
|
||||
uv run python -m pipeline.download.broadband --output $@
|
||||
|
||||
$(POSTCODES):
|
||||
uv run python -m pipeline.download.postcodes --output $@
|
||||
|
||||
$(GEOSURE_STAMP):
|
||||
uv run python -m pipeline.download.geosure --output $(GEOSURE_DIR)
|
||||
@touch $@
|
||||
|
||||
$(NOISE): $(ARCGIS)
|
||||
uv run python -m pipeline.download.noise --arcgis $(ARCGIS) --output $@
|
||||
|
||||
$(INSPIRE_STAMP):
|
||||
uv run python -m pipeline.download.inspire --output $(INSPIRE_DIR)
|
||||
@touch $@
|
||||
|
||||
$(OA_BOUNDARIES):
|
||||
uv run python -m pipeline.download.oa_boundaries --output $@
|
||||
|
||||
$(UPRN_LOOKUP):
|
||||
uv run python -m pipeline.download.uprn_lookup --output $@
|
||||
|
||||
# ── Journey times (requires TFL_API_KEY) ──────────────────────────────────────
|
||||
|
||||
$(JT_BANK):
|
||||
@if [ -f "$(MANUAL_DATA)/journey_times_bank.parquet" ]; then \
|
||||
echo "Copying journey_times_bank.parquet from manual_data/"; \
|
||||
cp "$(MANUAL_DATA)/journey_times_bank.parquet" $@; \
|
||||
else \
|
||||
echo ""; \
|
||||
echo "=== TFL journey times (bank) not found ==="; \
|
||||
echo "Either place the file in $(MANUAL_DATA)/journey_times_bank.parquet"; \
|
||||
echo "or register for a TFL API key at https://api-portal.tfl.gov.uk/signin"; \
|
||||
echo "and run: TFL_API_KEY=... make -f Makefile.data journey-times DEST=bank"; \
|
||||
echo ""; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
$(JT_FITZROVIA):
|
||||
@if [ -f "$(MANUAL_DATA)/journey_times_fitzrovia.parquet" ]; then \
|
||||
echo "Copying journey_times_fitzrovia.parquet from manual_data/"; \
|
||||
cp "$(MANUAL_DATA)/journey_times_fitzrovia.parquet" $@; \
|
||||
else \
|
||||
echo ""; \
|
||||
echo "=== TFL journey times (fitzrovia) not found ==="; \
|
||||
echo "Either place the file in $(MANUAL_DATA)/journey_times_fitzrovia.parquet"; \
|
||||
echo "or register for a TFL API key at https://api-portal.tfl.gov.uk/signin"; \
|
||||
echo "and run: TFL_API_KEY=... make -f Makefile.data journey-times DEST=fitzrovia"; \
|
||||
echo ""; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
journey-times: $(ARCGIS)
|
||||
ifndef DEST
|
||||
$(error DEST required — e.g. make journey-times DEST=bank)
|
||||
endif
|
||||
uv run python -m pipeline.journey_times --destination $(DEST) --output-dir $(DATA_DIR) --postcodes $(ARCGIS)
|
||||
|
||||
# ── Transforms ────────────────────────────────────────────────────────────────
|
||||
|
||||
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN)
|
||||
uv run python -m pipeline.transform.transform_poi --input $(POIS_RAW) --naptan $(NAPTAN) --output $@
|
||||
|
||||
$(EPC_PP): $(PRICE_PAID) $(EPC)
|
||||
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
||||
|
||||
$(CRIME):
|
||||
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@
|
||||
|
||||
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED)
|
||||
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --output $@
|
||||
|
||||
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS)
|
||||
uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
|
||||
|
||||
$(GEOSURE): $(GEOSURE_STAMP) $(ARCGIS)
|
||||
uv run python -m pipeline.transform.transform_geosure --geosure $(GEOSURE_DIR) --arcgis $(ARCGIS) --output $@
|
||||
|
||||
$(PC_BOUNDARIES): $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP)
|
||||
uv run python -m pipeline.transform.postcode_boundaries \
|
||||
--uprn $(UPRN_LOOKUP) \
|
||||
--oa-boundaries $(OA_BOUNDARIES) \
|
||||
--inspire $(INSPIRE_DIR) \
|
||||
--output $@
|
||||
|
||||
# ── Final merge ───────────────────────────────────────────────────────────────
|
||||
|
||||
$(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) \
|
||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE)
|
||||
uv run python -m pipeline.transform.merge \
|
||||
--epc-pp $(EPC_PP) \
|
||||
--arcgis $(ARCGIS) \
|
||||
--iod $(IOD) \
|
||||
--poi-proximity $(POI_PROXIMITY) \
|
||||
--journey-times-bank $(JT_BANK) \
|
||||
--journey-times-fitzrovia $(JT_FITZROVIA) \
|
||||
--ethnicity $(ETHNICITY) \
|
||||
--crime $(CRIME) \
|
||||
--noise $(NOISE) \
|
||||
--school-proximity $(SCHOOL_PROX) \
|
||||
--broadband $(BROADBAND) \
|
||||
--geosure $(GEOSURE) \
|
||||
--output $@
|
||||
62
README.md
62
README.md
|
|
@ -34,20 +34,40 @@ epc oopt out https://www.gov.uk/guidance/energy-performance-certificates-opt-out
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
We mapped every neighbourhood in England. You're welcome.
|
||||
|
||||
Harness our supercharged data to find your <green>perfect postcode</green>
|
||||
|
||||
|
||||
Set the interactive filters to match your preferences and watch the hidden gems get uncovered - instantly. Only you know your preferences. We will give you all the data that exists, aggregated into dozens of personalised sliders for you to compile the list of perfect postcodes.
|
||||
|
||||
<example with two sliders, last known price, minor crime/>
|
||||
|
||||
Now imagine this, but with 43 different filters across noise levels, environment risk, nearby amenities, demographics, socioeconomics, crime rate, transport link, and properties. (show the filter types with small cards).
|
||||
|
||||
|
||||
We strongly believe that the smart way to buy a property is by first understanding your expectations.
|
||||
|
||||
|
||||
|
||||
We give you the best-in-class tools to be intentional about the trade-offs you make in one of the most important decisions of your life.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
You may buy a box of cereal you’ve never tried because it’s 20% off. Your future home is not a box of cereal. Don’t let a seemingly good deal that just popped up turn into life-long regret because you got a discount but not on the home you needed. We help you reverse the equation and allow you to be intentional in your search. Don’t wait for good deals to come your way, if you can go out and find them yourself.
|
||||
|
||||
|
||||
You can’t get a complete picture of the property market by looking at current listings. To understand the landscape, you have to look at historical trends.
|
||||
There are too few properties listed at any time to give you a full and nuanced picture of the entire property landscape but there’re too many to go through one by one and evaluate them all. You can’t get a complete picture of the property market by looking at current listings. To understand the landscape, you have to look at historical trends. Don’t let the market sway you, anchor your expectations based on the choices millions of other buyers who’ve been in the same situation as you are now.
|
||||
|
||||
We give you the tools to be intentional about the trade-offs you make in one of the most important decisions of your life.
|
||||
|
||||
Don’t let the market sway you, anchor your expectations based on the choices millions of other buyers who’ve been in the same situation as you are now.
|
||||
|
||||
There are too few properties listed at any time to give you a full and nuanced picture of the entire property landscape but there’re too many to go through one by one and evaluate them all.
|
||||
|
||||
We all care about different things in our homes and living environments. Some of us are weary of noise and would like to avoid living next to a loud airfield as much as possible. And some of us are avid plane spotters.
|
||||
|
||||
Only you know your preferences. We will give you all the data that exists, aggregated into dozens of personalised sliders for you to compile the list of perfect postcodes.
|
||||
|
||||
We will help you find the best places to live within your budget regardless if there’s a property listed there right now. The best things come to those who’re patient. We will justify your patience. But we will also show you if your expectations are impossible to meet. I’d much rather be told upfront then spend months of my life looking for something that can’t possibly exist.
|
||||
|
||||
|
|
@ -58,18 +78,13 @@ We give you all the data and tools to become an Well-informed Buyer through the
|
|||
|
||||
---
|
||||
|
||||
- scraping
|
||||
- fix frontend
|
||||
- map hexagons
|
||||
- dragging
|
||||
- account management
|
||||
- stripe
|
||||
- update texts
|
||||
- fix plausible
|
||||
- move data to raid
|
||||
- extract all user-facing texts into a yaml file for easy editing
|
||||
|
||||
- register domain
|
||||
- register for email
|
||||
|
||||
|
||||
|
|
@ -86,33 +101,12 @@ FAQ:
|
|||
## outstadning prompts
|
||||
|
||||
|
||||
Add licensing to the app. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too. While doing this, protect the server against DOS-ing.
|
||||
|
||||
Add licensing to the app. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too.
|
||||
|
||||
-
|
||||
- the area stastics are missing for postcodes, they only work for hexagons
|
||||
- in the mobile view, move the property density and previewing colour spectrum to the bottom half of the screen.
|
||||
- make the no active filters have less padding on phone
|
||||
- add blue/green rollout
|
||||
- rename OgImageQuery to ScreenshotQuery
|
||||
- make the eye and plus icons and their touch targets twice the size
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## name ideas
|
||||
|
||||
|
||||
perfect postcodes
|
||||
|
||||
golden postcodes
|
||||
|
||||
calculated move
|
||||
|
||||
the spec
|
||||
|
||||
geologic
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ services:
|
|||
screenshot:
|
||||
build: /volumes/syncthing/Projects/property-map/screenshot
|
||||
environment:
|
||||
NARROWIT_URL: http://frontend:3001
|
||||
APP_URL: http://frontend:3001
|
||||
CACHE_DIR: /cache
|
||||
volumes:
|
||||
- screenshot-cache:/cache
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ const FAQ_ITEMS: FAQItem[] = [
|
|||
{
|
||||
question: 'What is this application?',
|
||||
answer:
|
||||
'Narrowit is an interactive map that visualises property-level data across England and Wales. It combines Land Registry sale prices, EPC energy certificates, TfL journey times, deprivation indices, crime statistics, broadband speeds, school ratings, road noise levels, ethnicity demographics, and OpenStreetMap points of interest into a single explorable view.',
|
||||
'Perfect Postcodes is an interactive map that visualises property-level data across England and Wales. It combines Land Registry sale prices, EPC energy certificates, TfL journey times, deprivation indices, crime statistics, broadband speeds, school ratings, road noise levels, ethnicity demographics, and OpenStreetMap points of interest into a single explorable view.',
|
||||
},
|
||||
{
|
||||
question: 'Where does the data come from?',
|
||||
|
|
@ -101,7 +101,7 @@ export default function FAQPage() {
|
|||
Frequently Asked Questions
|
||||
</h1>
|
||||
<p className="text-warm-600 dark:text-warm-400 mb-6">
|
||||
Common questions about how Narrowit works, where the data comes from, and how to use the
|
||||
Common questions about how Perfect Postcodes works, where the data comes from, and how to use the
|
||||
map.
|
||||
</p>
|
||||
<div className="space-y-3">
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ export default function HomePage({
|
|||
</h1>
|
||||
<p className="text-xl text-warm-600 dark:text-warm-400 mb-8 leading-relaxed max-w-xl">
|
||||
Set the commute, budget, school rating, noise level, and crime threshold you'll
|
||||
accept. Narrowit shows you every area that qualifies — instantly.
|
||||
accept. Perfect Postcodes shows you every area that qualifies — instantly.
|
||||
</p>
|
||||
<div className="flex items-center gap-4">
|
||||
<button
|
||||
|
|
@ -89,7 +89,7 @@ export default function HomePage({
|
|||
</div>
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold text-teal-600 uppercase tracking-wide mb-2">
|
||||
With Narrowit
|
||||
With Perfect Postcodes
|
||||
</h3>
|
||||
<p className="text-warm-700 dark:text-warm-300 leading-relaxed">
|
||||
Tell the map what you need. Every hexagon that lights up is a place worth
|
||||
|
|
|
|||
58
frontend/src/components/map/AISummaryCard.tsx
Normal file
58
frontend/src/components/map/AISummaryCard.tsx
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { ChevronIcon } from '../ui/icons';
|
||||
import { LightbulbIcon } from '../ui/icons/LightbulbIcon';
|
||||
|
||||
interface AISummaryCardProps {
|
||||
summary?: string;
|
||||
loading?: boolean;
|
||||
error?: string | null;
|
||||
expanded: boolean;
|
||||
onToggleExpanded: () => void;
|
||||
}
|
||||
|
||||
export default function AISummaryCard({
|
||||
summary,
|
||||
loading,
|
||||
error,
|
||||
expanded,
|
||||
onToggleExpanded,
|
||||
}: AISummaryCardProps) {
|
||||
if (!summary && !loading && !error) return null;
|
||||
|
||||
return (
|
||||
<div className="px-3 pt-3 pb-1">
|
||||
<div className="bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5">
|
||||
<button
|
||||
onClick={onToggleExpanded}
|
||||
className="w-full flex items-center justify-between gap-1.5 mb-1.5"
|
||||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<LightbulbIcon className="w-3.5 h-3.5 text-teal-600 dark:text-teal-400" />
|
||||
<span className="text-xs font-semibold text-teal-700 dark:text-teal-400">
|
||||
AI Summary
|
||||
</span>
|
||||
</div>
|
||||
<ChevronIcon
|
||||
direction={expanded ? 'down' : 'right'}
|
||||
className="w-3.5 h-3.5 text-teal-600 dark:text-teal-400"
|
||||
/>
|
||||
</button>
|
||||
{expanded && (
|
||||
<>
|
||||
{error ? (
|
||||
<div className="text-xs text-warm-600 dark:text-warm-400">
|
||||
Failed to generate summary.
|
||||
</div>
|
||||
) : loading ? (
|
||||
<div className="space-y-1.5">
|
||||
<div className="h-3 bg-teal-200/60 dark:bg-teal-800/40 rounded animate-pulse w-full" />
|
||||
<div className="h-3 bg-teal-200/60 dark:bg-teal-800/40 rounded animate-pulse w-4/5" />
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-xs text-warm-700 dark:text-warm-300 leading-relaxed">{summary}</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { useMemo, useState } from 'react';
|
||||
import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups';
|
||||
import type {
|
||||
FeatureFilters,
|
||||
FeatureMeta,
|
||||
|
|
@ -15,13 +16,15 @@ import StackedBarChart from './StackedBarChart';
|
|||
import StackedEnumChart from './StackedEnumChart';
|
||||
import PriceHistoryChart from './PriceHistoryChart';
|
||||
import ExternalSearchLinks from './ExternalSearchLinks';
|
||||
import { InfoIcon, CloseIcon, ChevronIcon } from '../ui/icons';
|
||||
import { InfoIcon, CloseIcon } from '../ui/icons';
|
||||
import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader';
|
||||
import { LightbulbIcon } from '../ui/icons/LightbulbIcon';
|
||||
import { IconButton } from '../ui/IconButton';
|
||||
import { FeatureInfoPopup } from '../ui/FeatureInfoPopup';
|
||||
import { EmptyState } from '../ui/EmptyState';
|
||||
import { FeatureLabel } from '../ui/FeatureLabel';
|
||||
import AISummaryCard from './AISummaryCard';
|
||||
import StreetViewEmbed from './StreetViewEmbed';
|
||||
import HistogramLegend from './HistogramLegend';
|
||||
|
||||
interface AreaPaneProps {
|
||||
stats: HexagonStatsResponse | null;
|
||||
|
|
@ -60,17 +63,9 @@ export default function AreaPane({
|
|||
const propertyCount = isPostcode && postcodeData ? postcodeData.properties.count : stats?.count;
|
||||
const featureGroups = useMemo(() => groupFeaturesByCategory(globalFeatures), [globalFeatures]);
|
||||
const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null);
|
||||
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
|
||||
const [collapsedGroups, toggleGroup] = useCollapsibleGroups();
|
||||
const [aiSummaryExpanded, setAiSummaryExpanded] = useState(true);
|
||||
|
||||
const toggleGroup = (name: string) =>
|
||||
setCollapsedGroups((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(name)) next.delete(name);
|
||||
else next.add(name);
|
||||
return next;
|
||||
});
|
||||
|
||||
const numericByName = useMemo(() => {
|
||||
if (!stats) return new Map();
|
||||
return new Map(stats.numeric_features.map((feature) => [feature.name, feature]));
|
||||
|
|
@ -138,78 +133,18 @@ export default function AreaPane({
|
|||
)}
|
||||
|
||||
<div className="flex-1 overflow-y-auto">
|
||||
{/* AI Summary Card */}
|
||||
{(aiSummary || aiSummaryLoading || aiSummaryError) && (
|
||||
<div className="px-3 pt-3 pb-1">
|
||||
<div className="bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5">
|
||||
<button
|
||||
onClick={() => setAiSummaryExpanded(!aiSummaryExpanded)}
|
||||
className="w-full flex items-center justify-between gap-1.5 mb-1.5"
|
||||
>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<LightbulbIcon className="w-3.5 h-3.5 text-teal-600 dark:text-teal-400" />
|
||||
<span className="text-xs font-semibold text-teal-700 dark:text-teal-400">
|
||||
AI Summary
|
||||
</span>
|
||||
</div>
|
||||
<ChevronIcon
|
||||
direction={aiSummaryExpanded ? 'down' : 'right'}
|
||||
className="w-3.5 h-3.5 text-teal-600 dark:text-teal-400"
|
||||
/>
|
||||
</button>
|
||||
{aiSummaryExpanded && (
|
||||
<>
|
||||
{aiSummaryError ? (
|
||||
<div className="text-xs text-warm-600 dark:text-warm-400">
|
||||
Failed to generate summary.
|
||||
</div>
|
||||
) : aiSummaryLoading ? (
|
||||
<div className="space-y-1.5">
|
||||
<div className="h-3 bg-teal-200/60 dark:bg-teal-800/40 rounded animate-pulse w-full" />
|
||||
<div className="h-3 bg-teal-200/60 dark:bg-teal-800/40 rounded animate-pulse w-4/5" />
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-xs text-warm-700 dark:text-warm-300 leading-relaxed">
|
||||
{aiSummary}
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<AISummaryCard
|
||||
summary={aiSummary}
|
||||
loading={aiSummaryLoading}
|
||||
error={aiSummaryError}
|
||||
expanded={aiSummaryExpanded}
|
||||
onToggleExpanded={() => setAiSummaryExpanded(!aiSummaryExpanded)}
|
||||
/>
|
||||
{loading && !stats ? (
|
||||
<LoadingSkeleton />
|
||||
) : stats ? (
|
||||
<div>
|
||||
{/* Histogram color legend */}
|
||||
<div className="mx-3 mt-3 bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5 text-xs">
|
||||
<div className="space-y-1.5">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-3 bg-teal-500 dark:bg-teal-400 rounded" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">Teal bars</span>{' '}
|
||||
show the distribution in this selected area
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-3 bg-warm-300/60 dark:bg-warm-600/60 rounded" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">Gray bars</span>{' '}
|
||||
show the overall distribution across all areas
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-px border-t border-dashed border-warm-500 dark:border-warm-400" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">
|
||||
Dashed line
|
||||
</span>{' '}
|
||||
indicates the global average
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<HistogramLegend />
|
||||
{featureGroups.map((group) => {
|
||||
const hasData = group.features.some(
|
||||
(feature) => numericByName.has(feature.name) || enumByName.has(feature.name)
|
||||
|
|
@ -460,25 +395,7 @@ export default function AreaPane({
|
|||
</div>
|
||||
);
|
||||
})}
|
||||
{/* Google Street View */}
|
||||
{hexagonLocation && (
|
||||
<div>
|
||||
<div className="px-3 py-1.5 text-xs font-bold text-warm-500 bg-warm-50 dark:bg-warm-900 dark:text-warm-400 sticky top-0">
|
||||
Street View
|
||||
</div>
|
||||
<div className="px-3 py-2">
|
||||
<div className="rounded overflow-hidden border border-warm-200 dark:border-warm-700">
|
||||
<iframe
|
||||
className="w-full"
|
||||
style={{ height: 240, border: 0 }}
|
||||
loading="lazy"
|
||||
referrerPolicy="no-referrer-when-downgrade"
|
||||
src={`https://maps.google.com/maps?layer=c&cbll=${hexagonLocation.lat},${hexagonLocation.lon}&cbp=11,0,0,0,0&output=svembed`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{hexagonLocation && <StreetViewEmbed location={hexagonLocation} />}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
|
|
|||
129
frontend/src/components/map/FeatureBrowser.tsx
Normal file
129
frontend/src/components/map/FeatureBrowser.tsx
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import { useState, useMemo, useEffect } from 'react';
|
||||
import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups';
|
||||
import { SearchInput } from '../ui/SearchInput';
|
||||
import { FilterIcon } from '../ui/icons';
|
||||
import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader';
|
||||
import { EmptyState } from '../ui/EmptyState';
|
||||
import type { FeatureMeta } from '../../types';
|
||||
import { groupFeaturesByCategory } from '../../lib/features';
|
||||
import { FeatureInfoPopup } from '../ui/FeatureInfoPopup';
|
||||
import { FeatureActions } from '../ui/FeatureIcons';
|
||||
import { FeatureLabel } from '../ui/FeatureLabel';
|
||||
|
||||
interface FeatureBrowserProps {
|
||||
availableFeatures: FeatureMeta[];
|
||||
allFeatures: FeatureMeta[];
|
||||
pinnedFeature: string | null;
|
||||
onAddFilter: (name: string) => void;
|
||||
onTogglePin: (name: string) => void;
|
||||
onNavigateToSource?: (slug: string, featureName: string) => void;
|
||||
openInfoFeature?: string | null;
|
||||
onClearOpenInfoFeature?: () => void;
|
||||
}
|
||||
|
||||
export default function FeatureBrowser({
|
||||
availableFeatures,
|
||||
allFeatures,
|
||||
pinnedFeature,
|
||||
onAddFilter,
|
||||
onTogglePin,
|
||||
onNavigateToSource,
|
||||
openInfoFeature,
|
||||
onClearOpenInfoFeature,
|
||||
}: FeatureBrowserProps) {
|
||||
const [search, setSearch] = useState('');
|
||||
const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null);
|
||||
const [expandedGroups, toggleGroup] = useCollapsibleGroups();
|
||||
|
||||
useEffect(() => {
|
||||
if (openInfoFeature) {
|
||||
const feat = allFeatures.find((f) => f.name === openInfoFeature);
|
||||
if (feat) setInfoFeature(feat);
|
||||
onClearOpenInfoFeature?.();
|
||||
}
|
||||
}, [openInfoFeature, allFeatures, onClearOpenInfoFeature]);
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
if (!search) return availableFeatures;
|
||||
const lower = search.toLowerCase();
|
||||
return availableFeatures.filter((f) => f.name.toLowerCase().includes(lower));
|
||||
}, [availableFeatures, search]);
|
||||
|
||||
const grouped = useMemo(() => groupFeaturesByCategory(filtered), [filtered]);
|
||||
|
||||
// When searching, expand all groups so results are visible
|
||||
const isSearching = search.length > 0;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="shrink-0 p-2 border-b border-warm-200 dark:border-navy-700">
|
||||
<SearchInput value={search} onChange={setSearch} placeholder="Search features..." />
|
||||
</div>
|
||||
<div className="md:min-h-0 md:flex-1 md:overflow-y-auto flex flex-col">
|
||||
{grouped.map((group) => {
|
||||
const isExpanded = isSearching || expandedGroups.has(group.name);
|
||||
return (
|
||||
<div key={group.name} className="shrink-0">
|
||||
<CollapsibleGroupHeader
|
||||
name={group.name}
|
||||
expanded={isExpanded}
|
||||
onToggle={() => toggleGroup(group.name)}
|
||||
className="px-3 py-1.5 text-xs font-bold text-warm-500 bg-warm-50 dark:bg-navy-950 dark:text-warm-400 sticky top-0 hover:bg-warm-100 dark:hover:bg-warm-800"
|
||||
>
|
||||
<span className="text-[10px] font-medium text-warm-400 dark:text-warm-500">
|
||||
{group.features.length}
|
||||
</span>
|
||||
</CollapsibleGroupHeader>
|
||||
{isExpanded &&
|
||||
group.features.map((f) => {
|
||||
const isPinned = pinnedFeature === f.name;
|
||||
return (
|
||||
<div
|
||||
key={f.name}
|
||||
className="flex items-start justify-between px-3 py-1.5 hover:bg-teal-50 dark:hover:bg-teal-900/30 dark:text-warm-300"
|
||||
>
|
||||
<div className="min-w-0 mr-2">
|
||||
<FeatureLabel feature={f} onShowInfo={setInfoFeature} size="sm" />
|
||||
{f.description && (
|
||||
<span className="text-xs text-warm-400 dark:text-warm-500 truncate block">
|
||||
{f.description}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<FeatureActions
|
||||
feature={f}
|
||||
isPinned={isPinned}
|
||||
onTogglePin={onTogglePin}
|
||||
onAdd={onAddFilter}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{grouped.length === 0 ? (
|
||||
<EmptyState
|
||||
icon={<FilterIcon className="w-8 h-8 text-warm-300 dark:text-warm-600" />}
|
||||
title={search ? 'No matching features' : 'All features are active'}
|
||||
description={
|
||||
search ? 'Try a different search term' : 'Remove a filter to see available features'
|
||||
}
|
||||
className="px-3 py-4"
|
||||
/>
|
||||
) : (
|
||||
<p className="flex-1 flex items-center justify-center px-4 py-6 text-xs text-center text-warm-400 dark:text-warm-500">
|
||||
Everyone cares about different things. Pick the filters that matter most to you.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{infoFeature && (
|
||||
<FeatureInfoPopup
|
||||
feature={infoFeature}
|
||||
onClose={() => setInfoFeature(null)}
|
||||
onNavigateToSource={onNavigateToSource}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,16 +1,14 @@
|
|||
import { memo, useState, useMemo, useEffect } from 'react';
|
||||
import { memo, useState } from 'react';
|
||||
import { Slider } from '../ui/Slider';
|
||||
import { SearchInput } from '../ui/SearchInput';
|
||||
import { FilterIcon, LightbulbIcon } from '../ui/icons';
|
||||
import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader';
|
||||
import { EmptyState } from '../ui/EmptyState';
|
||||
import type { FeatureMeta, FeatureFilters } from '../../types';
|
||||
import { formatFilterValue } from '../../lib/format';
|
||||
import { groupFeaturesByCategory } from '../../lib/features';
|
||||
import InfoPopup from '../ui/InfoPopup';
|
||||
import { FeatureInfoPopup } from '../ui/FeatureInfoPopup';
|
||||
import { FeatureActions } from '../ui/FeatureIcons';
|
||||
import { FeatureLabel } from '../ui/FeatureLabel';
|
||||
import FeatureBrowser from './FeatureBrowser';
|
||||
|
||||
interface FiltersProps {
|
||||
features: FeatureMeta[];
|
||||
|
|
@ -35,130 +33,6 @@ interface FiltersProps {
|
|||
onClearOpenInfoFeature?: () => void;
|
||||
}
|
||||
|
||||
function FeatureBrowser({
|
||||
availableFeatures,
|
||||
allFeatures,
|
||||
pinnedFeature,
|
||||
onAddFilter,
|
||||
onTogglePin,
|
||||
onNavigateToSource,
|
||||
openInfoFeature,
|
||||
onClearOpenInfoFeature,
|
||||
}: {
|
||||
availableFeatures: FeatureMeta[];
|
||||
allFeatures: FeatureMeta[];
|
||||
pinnedFeature: string | null;
|
||||
onAddFilter: (name: string) => void;
|
||||
onTogglePin: (name: string) => void;
|
||||
onNavigateToSource?: (slug: string, featureName: string) => void;
|
||||
openInfoFeature?: string | null;
|
||||
onClearOpenInfoFeature?: () => void;
|
||||
}) {
|
||||
const [search, setSearch] = useState('');
|
||||
const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null);
|
||||
const [expandedGroups, setExpandedGroups] = useState<Set<string>>(new Set());
|
||||
|
||||
const toggleGroup = (name: string) =>
|
||||
setExpandedGroups((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(name)) next.delete(name);
|
||||
else next.add(name);
|
||||
return next;
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (openInfoFeature) {
|
||||
const feat = allFeatures.find((f) => f.name === openInfoFeature);
|
||||
if (feat) setInfoFeature(feat);
|
||||
onClearOpenInfoFeature?.();
|
||||
}
|
||||
}, [openInfoFeature, allFeatures, onClearOpenInfoFeature]);
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
if (!search) return availableFeatures;
|
||||
const lower = search.toLowerCase();
|
||||
return availableFeatures.filter((f) => f.name.toLowerCase().includes(lower));
|
||||
}, [availableFeatures, search]);
|
||||
|
||||
const grouped = useMemo(() => groupFeaturesByCategory(filtered), [filtered]);
|
||||
|
||||
// When searching, expand all groups so results are visible
|
||||
const isSearching = search.length > 0;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="shrink-0 p-2 border-b border-warm-200 dark:border-navy-700">
|
||||
<SearchInput value={search} onChange={setSearch} placeholder="Search features..." />
|
||||
</div>
|
||||
<div className="md:min-h-0 md:flex-1 md:overflow-y-auto flex flex-col">
|
||||
{grouped.map((group) => {
|
||||
const isExpanded = isSearching || expandedGroups.has(group.name);
|
||||
return (
|
||||
<div key={group.name} className="shrink-0">
|
||||
<CollapsibleGroupHeader
|
||||
name={group.name}
|
||||
expanded={isExpanded}
|
||||
onToggle={() => toggleGroup(group.name)}
|
||||
className="px-3 py-1.5 text-xs font-bold text-warm-500 bg-warm-50 dark:bg-navy-950 dark:text-warm-400 sticky top-0 hover:bg-warm-100 dark:hover:bg-warm-800"
|
||||
>
|
||||
<span className="text-[10px] font-medium text-warm-400 dark:text-warm-500">
|
||||
{group.features.length}
|
||||
</span>
|
||||
</CollapsibleGroupHeader>
|
||||
{isExpanded &&
|
||||
group.features.map((f) => {
|
||||
const isPinned = pinnedFeature === f.name;
|
||||
return (
|
||||
<div
|
||||
key={f.name}
|
||||
className="flex items-start justify-between px-3 py-1.5 hover:bg-teal-50 dark:hover:bg-teal-900/30 dark:text-warm-300"
|
||||
>
|
||||
<div className="min-w-0 mr-2">
|
||||
<FeatureLabel feature={f} onShowInfo={setInfoFeature} size="sm" />
|
||||
{f.description && (
|
||||
<span className="text-xs text-warm-400 dark:text-warm-500 truncate block">
|
||||
{f.description}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<FeatureActions
|
||||
feature={f}
|
||||
isPinned={isPinned}
|
||||
onTogglePin={onTogglePin}
|
||||
onAdd={onAddFilter}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{grouped.length === 0 ? (
|
||||
<EmptyState
|
||||
icon={<FilterIcon className="w-8 h-8 text-warm-300 dark:text-warm-600" />}
|
||||
title={search ? 'No matching features' : 'All features are active'}
|
||||
description={
|
||||
search ? 'Try a different search term' : 'Remove a filter to see available features'
|
||||
}
|
||||
className="px-3 py-4"
|
||||
/>
|
||||
) : (
|
||||
<p className="flex-1 flex items-center justify-center px-4 py-6 text-xs text-center text-warm-400 dark:text-warm-500">
|
||||
Everyone cares about different things. Pick the filters that matter most to you.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{infoFeature && (
|
||||
<FeatureInfoPopup
|
||||
feature={infoFeature}
|
||||
onClose={() => setInfoFeature(null)}
|
||||
onNavigateToSource={onNavigateToSource}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default memo(function Filters({
|
||||
features,
|
||||
filters,
|
||||
|
|
|
|||
29
frontend/src/components/map/HistogramLegend.tsx
Normal file
29
frontend/src/components/map/HistogramLegend.tsx
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
export default function HistogramLegend() {
|
||||
return (
|
||||
<div className="mx-3 mt-3 bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5 text-xs">
|
||||
<div className="space-y-1.5">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-3 bg-teal-500 dark:bg-teal-400 rounded" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">Teal bars</span> show the
|
||||
distribution in this selected area
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-3 bg-warm-300/60 dark:bg-warm-600/60 rounded" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">Gray bars</span> show the
|
||||
overall distribution across all areas
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="w-3 h-px border-t border-dashed border-warm-500 dark:border-warm-400" />
|
||||
<span className="text-warm-700 dark:text-warm-300">
|
||||
<span className="font-medium text-warm-900 dark:text-warm-100">Dashed line</span>{' '}
|
||||
indicates the global average
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -2,14 +2,10 @@ import { useCallback, useRef, useEffect, useState, useMemo, memo } from 'react';
|
|||
import { Map as MapGL, useControl } from 'react-map-gl/maplibre';
|
||||
import type { MapRef } from 'react-map-gl/maplibre';
|
||||
import { MapboxOverlay } from '@deck.gl/mapbox';
|
||||
import { H3HexagonLayer } from '@deck.gl/geo-layers';
|
||||
import { GeoJsonLayer, IconLayer, TextLayer } from '@deck.gl/layers';
|
||||
import type { PickingInfo } from '@deck.gl/core';
|
||||
import 'maplibre-gl/dist/maplibre-gl.css';
|
||||
import type {
|
||||
HexagonData,
|
||||
PostcodeFeature,
|
||||
PostcodeProperties,
|
||||
ViewState,
|
||||
ViewChangeParams,
|
||||
POI,
|
||||
|
|
@ -17,30 +13,13 @@ import type {
|
|||
Bounds,
|
||||
} from '../../types';
|
||||
|
||||
import {
|
||||
GRADIENT,
|
||||
normalizedToColor,
|
||||
countToColor,
|
||||
zoomToResolution,
|
||||
getBoundsFromViewState,
|
||||
emojiToTwemojiUrl,
|
||||
getMapStyle,
|
||||
DENSITY_GRADIENT,
|
||||
DENSITY_GRADIENT_DARK,
|
||||
} from '../../lib/map-utils';
|
||||
import { zoomToResolution, getBoundsFromViewState, getMapStyle } from '../../lib/map-utils';
|
||||
import { INITIAL_VIEW_STATE, MAP_MIN_ZOOM, MAP_BOUNDS } from '../../lib/consts';
|
||||
import PostcodeSearch, { type SearchedPostcode } from './PostcodeSearch';
|
||||
import MapLegend from './MapLegend';
|
||||
import HoverCard from './HoverCard';
|
||||
import type { FeatureFilters } from '../../types';
|
||||
|
||||
/** Convert POI id (e.g. "n12345") to OpenStreetMap URL */
|
||||
function osmIdToUrl(id: string): string | null {
|
||||
const match = id.match(/^([nwr])(\d+)$/);
|
||||
if (!match) return null;
|
||||
const typeMap: Record<string, string> = { n: 'node', w: 'way', r: 'relation' };
|
||||
return `https://www.openstreetmap.org/${typeMap[match[1]]}/${match[2]}`;
|
||||
}
|
||||
import { useDeckLayers, osmIdToUrl } from '../../hooks/useDeckLayers';
|
||||
|
||||
interface MapProps {
|
||||
data: HexagonData[];
|
||||
|
|
@ -123,7 +102,6 @@ export default memo(function Map({
|
|||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const [viewState, setViewState] = useState<ViewState>(initialViewState || INITIAL_VIEW_STATE);
|
||||
const [dimensions, setDimensions] = useState<Dimensions>({ width: 0, height: 0 });
|
||||
const [hoverPosition, setHoverPosition] = useState<{ x: number; y: number } | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
|
|
@ -165,9 +143,6 @@ export default memo(function Map({
|
|||
setViewState((prev) => ({ ...prev, latitude: lat, longitude: lng, zoom }));
|
||||
}, []);
|
||||
|
||||
const themeRef = useRef(theme);
|
||||
themeRef.current = theme;
|
||||
|
||||
const handleMapLoad = useCallback(
|
||||
(_evt: { target: MapRef['getMap'] extends () => infer M ? M : never }) => {
|
||||
// Road opacity is set in getMapStyle
|
||||
|
|
@ -177,425 +152,31 @@ export default memo(function Map({
|
|||
|
||||
const mapStyle = useMemo(() => getMapStyle(theme), [theme]);
|
||||
|
||||
const [popupInfo, setPopupInfo] = useState<{
|
||||
x: number;
|
||||
y: number;
|
||||
name: string;
|
||||
category: string;
|
||||
id: string;
|
||||
} | null>(null);
|
||||
|
||||
const handlePoiHover = useCallback((info: PickingInfo<POI>) => {
|
||||
if (info.object && info.x !== undefined && info.y !== undefined) {
|
||||
setPopupInfo({
|
||||
x: info.x,
|
||||
y: info.y,
|
||||
name: info.object.name,
|
||||
category: info.object.category,
|
||||
id: info.object.id,
|
||||
});
|
||||
} else {
|
||||
setPopupInfo(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const countRange = useMemo(() => {
|
||||
if (data.length === 0) return { min: 0, max: 1 };
|
||||
let min = Infinity;
|
||||
let max = -Infinity;
|
||||
for (const d of data) {
|
||||
if (viewportBounds) {
|
||||
if (
|
||||
d.lat < viewportBounds.south ||
|
||||
d.lat > viewportBounds.north ||
|
||||
d.lon < viewportBounds.west ||
|
||||
d.lon > viewportBounds.east
|
||||
)
|
||||
continue;
|
||||
}
|
||||
const c = d.count as number;
|
||||
if (c < min) min = c;
|
||||
if (c > max) max = c;
|
||||
}
|
||||
if (min === Infinity) return { min: 0, max: 1 };
|
||||
if (min === max) return { min, max: min + 1 };
|
||||
return { min, max };
|
||||
}, [data, viewportBounds]);
|
||||
|
||||
const colorFeatureMeta = useMemo(
|
||||
() => (viewFeature ? features.find((f) => f.name === viewFeature) || null : null),
|
||||
[viewFeature, features]
|
||||
);
|
||||
|
||||
const viewFeatureRef = useRef(viewFeature);
|
||||
viewFeatureRef.current = viewFeature;
|
||||
const colorRangeRef = useRef(colorRange);
|
||||
colorRangeRef.current = colorRange;
|
||||
const filterRangeRef = useRef(filterRange);
|
||||
filterRangeRef.current = filterRange;
|
||||
const colorFeatureMetaRef = useRef(colorFeatureMeta);
|
||||
colorFeatureMetaRef.current = colorFeatureMeta;
|
||||
const countRangeRef = useRef(countRange);
|
||||
countRangeRef.current = countRange;
|
||||
const selectedHexagonIdRef = useRef(selectedHexagonId);
|
||||
selectedHexagonIdRef.current = selectedHexagonId;
|
||||
const hoveredHexagonIdRef = useRef(hoveredHexagonId);
|
||||
hoveredHexagonIdRef.current = hoveredHexagonId;
|
||||
|
||||
const onHexagonClickRef = useRef(onHexagonClick);
|
||||
onHexagonClickRef.current = onHexagonClick;
|
||||
const handleHexagonClick = useCallback((info: PickingInfo<HexagonData>) => {
|
||||
if (info.object && 'h3' in info.object) {
|
||||
onHexagonClickRef.current(info.object.h3);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const onHexagonHoverRef = useRef(onHexagonHover);
|
||||
onHexagonHoverRef.current = onHexagonHover;
|
||||
const handleHexagonHover = useCallback((info: PickingInfo<HexagonData>) => {
|
||||
if (info.object && 'h3' in info.object && info.x !== undefined && info.y !== undefined) {
|
||||
setHoverPosition({ x: info.x, y: info.y });
|
||||
onHexagonHoverRef.current(info.object.h3, info.x, info.y);
|
||||
} else {
|
||||
setHoverPosition(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handlePoiHoverRef = useRef(handlePoiHover);
|
||||
handlePoiHoverRef.current = handlePoiHover;
|
||||
const stablePoiHover = useCallback((info: PickingInfo<POI>) => {
|
||||
handlePoiHoverRef.current(info);
|
||||
}, []);
|
||||
|
||||
// Compute count range for postcodes (similar to hexagons)
|
||||
const postcodeCountRange = useMemo(() => {
|
||||
if (postcodeData.length === 0) return { min: 0, max: 1 };
|
||||
let min = Infinity;
|
||||
let max = -Infinity;
|
||||
for (const d of postcodeData) {
|
||||
if (viewportBounds) {
|
||||
const [lng, lat] = d.properties.centroid as [number, number];
|
||||
if (
|
||||
lat < viewportBounds.south ||
|
||||
lat > viewportBounds.north ||
|
||||
lng < viewportBounds.west ||
|
||||
lng > viewportBounds.east
|
||||
)
|
||||
continue;
|
||||
}
|
||||
const c = d.properties.count;
|
||||
if (c < min) min = c;
|
||||
if (c > max) max = c;
|
||||
}
|
||||
if (min === Infinity) return { min: 0, max: 1 };
|
||||
if (min === max) return { min, max: min + 1 };
|
||||
return { min, max };
|
||||
}, [postcodeData, viewportBounds]);
|
||||
|
||||
const postcodeCountRangeRef = useRef(postcodeCountRange);
|
||||
postcodeCountRangeRef.current = postcodeCountRange;
|
||||
|
||||
// Track selected/hovered postcode for styling
|
||||
const [selectedPostcode, setSelectedPostcode] = useState<string | null>(null);
|
||||
const [hoveredPostcode, setHoveredPostcode] = useState<string | null>(null);
|
||||
const selectedPostcodeRef = useRef(selectedPostcode);
|
||||
selectedPostcodeRef.current = selectedPostcode;
|
||||
const hoveredPostcodeRef = useRef(hoveredPostcode);
|
||||
hoveredPostcodeRef.current = hoveredPostcode;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handlePostcodeClick = useCallback((info: PickingInfo<any>) => {
|
||||
const pc = info.object?.properties?.postcode;
|
||||
if (pc) {
|
||||
setSelectedPostcode((prev) => (prev === pc ? null : pc));
|
||||
onHexagonClickRef.current(pc, true);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handlePostcodeHoverCallback = useCallback((info: PickingInfo<any>) => {
|
||||
const pc = info.object?.properties?.postcode;
|
||||
if (pc && info.x !== undefined && info.y !== undefined) {
|
||||
setHoveredPostcode(pc);
|
||||
setHoverPosition({ x: info.x, y: info.y });
|
||||
onHexagonHoverRef.current(pc, info.x, info.y);
|
||||
} else {
|
||||
setHoveredPostcode(null);
|
||||
setHoverPosition(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const isDark = theme === 'dark';
|
||||
const densityGradient = isDark ? DENSITY_GRADIENT_DARK : DENSITY_GRADIENT;
|
||||
const densityGradientRef = useRef(densityGradient);
|
||||
densityGradientRef.current = densityGradient;
|
||||
const isDarkRef = useRef(isDark);
|
||||
isDarkRef.current = isDark;
|
||||
|
||||
const colorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${countRange.min}|${countRange.max}|${selectedHexagonId}|${hoveredHexagonId}|${theme}`;
|
||||
const postcodeColorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${postcodeCountRange.min}|${postcodeCountRange.max}|${selectedPostcode}|${hoveredPostcode}|${theme}`;
|
||||
|
||||
const hexLayer = useMemo(
|
||||
() =>
|
||||
new H3HexagonLayer<HexagonData>({
|
||||
id: 'h3-hexagons',
|
||||
data,
|
||||
getHexagon: (d) => d.h3,
|
||||
getFillColor: (d) => {
|
||||
const vf = viewFeatureRef.current;
|
||||
const clr = colorRangeRef.current;
|
||||
const fr = filterRangeRef.current;
|
||||
const cfm = colorFeatureMetaRef.current;
|
||||
const dark = isDarkRef.current;
|
||||
if (vf && clr && cfm) {
|
||||
const val = d[`avg_${vf}`] ?? d[`min_${vf}`];
|
||||
if (val == null)
|
||||
return (dark ? [80, 70, 65, 80] : [128, 128, 128, 80]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
if (fr) {
|
||||
const minVal = d[`min_${vf}`] as number;
|
||||
const maxVal = d[`max_${vf}`] as number;
|
||||
if (maxVal < fr[0] || minVal > fr[1]) {
|
||||
return (dark ? [60, 55, 50, 60] : [180, 180, 180, 60]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
}
|
||||
const range = clr[1] - clr[0];
|
||||
if (range === 0) return [...GRADIENT[0].color, 255] as [number, number, number, number];
|
||||
const t = ((val as number) - clr[0]) / range;
|
||||
const rgb = normalizedToColor(Math.max(0, Math.min(1, t)));
|
||||
return [...rgb, 255] as [number, number, number, number];
|
||||
}
|
||||
const cr = countRangeRef.current;
|
||||
const c = d.count as number;
|
||||
const t = (c - cr.min) / (cr.max - cr.min);
|
||||
return [
|
||||
...countToColor(Math.max(0, Math.min(1, t)), densityGradientRef.current),
|
||||
255,
|
||||
] as [number, number, number, number];
|
||||
},
|
||||
getLineColor: (d) => {
|
||||
if (d.h3 === selectedHexagonIdRef.current)
|
||||
return [255, 255, 255, 255] as [number, number, number, number];
|
||||
if (d.h3 === hoveredHexagonIdRef.current)
|
||||
return [29, 228, 195, 200] as [number, number, number, number];
|
||||
return [0, 0, 0, 0] as [number, number, number, number];
|
||||
},
|
||||
getLineWidth: (d) => {
|
||||
if (d.h3 === selectedHexagonIdRef.current) return 3;
|
||||
if (d.h3 === hoveredHexagonIdRef.current) return 2;
|
||||
return 0;
|
||||
},
|
||||
lineWidthUnits: 'pixels',
|
||||
updateTriggers: {
|
||||
getFillColor: [colorTrigger],
|
||||
getLineColor: [colorTrigger],
|
||||
getLineWidth: [colorTrigger],
|
||||
},
|
||||
extruded: false,
|
||||
pickable: true,
|
||||
opacity: 1,
|
||||
highPrecision: true,
|
||||
onClick: handleHexagonClick,
|
||||
onHover: handleHexagonHover,
|
||||
// @ts-expect-error beforeId is a MapboxOverlay interleave prop, not typed in LayerProps
|
||||
beforeId: 'landuse_park',
|
||||
}),
|
||||
[data, colorTrigger, handleHexagonClick, handleHexagonHover]
|
||||
);
|
||||
|
||||
const postcodeLayer = useMemo(
|
||||
() =>
|
||||
new GeoJsonLayer<PostcodeProperties>({
|
||||
id: 'postcode-polygons',
|
||||
data: postcodeData as PostcodeFeature[],
|
||||
getFillColor: (f) => {
|
||||
const d = f.properties;
|
||||
const vf = viewFeatureRef.current;
|
||||
const clr = colorRangeRef.current;
|
||||
const fr = filterRangeRef.current;
|
||||
const cfm = colorFeatureMetaRef.current;
|
||||
const dark = isDarkRef.current;
|
||||
if (vf && clr && cfm) {
|
||||
const val = d[`avg_${vf}`] ?? d[`min_${vf}`];
|
||||
if (val == null)
|
||||
return (dark ? [80, 70, 65, 80] : [128, 128, 128, 80]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
if (fr) {
|
||||
const minVal = d[`min_${vf}`] as number;
|
||||
const maxVal = d[`max_${vf}`] as number;
|
||||
if (maxVal < fr[0] || minVal > fr[1]) {
|
||||
return (dark ? [60, 55, 50, 60] : [180, 180, 180, 60]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
}
|
||||
const range = clr[1] - clr[0];
|
||||
if (range === 0) return [...GRADIENT[0].color, 180] as [number, number, number, number];
|
||||
const t = ((val as number) - clr[0]) / range;
|
||||
const rgb = normalizedToColor(Math.max(0, Math.min(1, t)));
|
||||
return [...rgb, 180] as [number, number, number, number];
|
||||
}
|
||||
const cr = postcodeCountRangeRef.current;
|
||||
const c = d.count;
|
||||
const t = (c - cr.min) / (cr.max - cr.min);
|
||||
return [
|
||||
...countToColor(Math.max(0, Math.min(1, t)), densityGradientRef.current),
|
||||
180,
|
||||
] as [number, number, number, number];
|
||||
},
|
||||
getLineColor: (f) => {
|
||||
const pc = f.properties.postcode;
|
||||
const dark = isDarkRef.current;
|
||||
if (pc === selectedPostcodeRef.current)
|
||||
return [255, 255, 255, 255] as [number, number, number, number];
|
||||
if (pc === hoveredPostcodeRef.current)
|
||||
return [29, 228, 195, 200] as [number, number, number, number];
|
||||
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
},
|
||||
getLineWidth: (f) => {
|
||||
const pc = f.properties.postcode;
|
||||
if (pc === selectedPostcodeRef.current) return 3;
|
||||
if (pc === hoveredPostcodeRef.current) return 2;
|
||||
return 1;
|
||||
},
|
||||
lineWidthUnits: 'pixels',
|
||||
updateTriggers: {
|
||||
getFillColor: [postcodeColorTrigger],
|
||||
getLineColor: [postcodeColorTrigger],
|
||||
getLineWidth: [postcodeColorTrigger],
|
||||
},
|
||||
extruded: false,
|
||||
pickable: true,
|
||||
onClick: handlePostcodeClick,
|
||||
onHover: handlePostcodeHoverCallback,
|
||||
// @ts-expect-error beforeId is a MapboxOverlay interleave prop, not typed in LayerProps
|
||||
beforeId: 'landuse_park',
|
||||
}),
|
||||
[postcodeData, postcodeColorTrigger, handlePostcodeClick, handlePostcodeHoverCallback]
|
||||
);
|
||||
|
||||
const postcodeLabelsLayer = useMemo(
|
||||
() =>
|
||||
new TextLayer<PostcodeFeature>({
|
||||
id: 'postcode-labels',
|
||||
data: postcodeData,
|
||||
getPosition: (f) => f.properties.centroid,
|
||||
getText: (f) => f.properties.postcode,
|
||||
getSize: 12,
|
||||
getColor: theme === 'dark' ? [255, 255, 255, 240] : [40, 40, 40, 220],
|
||||
getTextAnchor: 'middle',
|
||||
getAlignmentBaseline: 'center',
|
||||
fontFamily: 'Inter, system-ui, sans-serif',
|
||||
fontWeight: 600,
|
||||
outlineWidth: 2,
|
||||
outlineColor: theme === 'dark' ? [30, 30, 30, 200] : [255, 255, 255, 200],
|
||||
sizeUnits: 'pixels',
|
||||
sizeMinPixels: 10,
|
||||
sizeMaxPixels: 14,
|
||||
billboard: false,
|
||||
pickable: false,
|
||||
}),
|
||||
[postcodeData, theme]
|
||||
);
|
||||
|
||||
const poiLayer = useMemo(
|
||||
() =>
|
||||
new IconLayer<POI>({
|
||||
id: 'poi-icons',
|
||||
data: pois,
|
||||
getPosition: (d) => [d.lng, d.lat],
|
||||
getIcon: (d) => ({
|
||||
url: emojiToTwemojiUrl(d.emoji),
|
||||
width: 72,
|
||||
height: 72,
|
||||
}),
|
||||
getSize: 24,
|
||||
sizeMinPixels: 20,
|
||||
sizeMaxPixels: 40,
|
||||
pickable: true,
|
||||
onHover: stablePoiHover,
|
||||
}),
|
||||
[pois, stablePoiHover]
|
||||
);
|
||||
|
||||
// Check if the searched postcode has data (passes current filters)
|
||||
const searchedPostcodeHasData = useMemo(() => {
|
||||
if (!searchedPostcode) return false;
|
||||
return postcodeData.some((f) => f.properties.postcode === searchedPostcode.postcode);
|
||||
}, [searchedPostcode, postcodeData]);
|
||||
|
||||
// Highlight layer for searched postcode
|
||||
const searchedPostcodeHighlightLayer = useMemo(() => {
|
||||
if (!searchedPostcode) return null;
|
||||
const hasData = searchedPostcodeHasData;
|
||||
const feature = {
|
||||
type: 'Feature' as const,
|
||||
geometry: searchedPostcode.geometry,
|
||||
properties: {},
|
||||
};
|
||||
return new GeoJsonLayer({
|
||||
id: 'searched-postcode-highlight',
|
||||
data: [feature],
|
||||
getFillColor: hasData
|
||||
? [29, 228, 195, 40] // teal tint when has data
|
||||
: [255, 180, 0, 30], // orange tint when filtered out
|
||||
getLineColor: hasData
|
||||
? [29, 228, 195, 255] // solid teal when has data
|
||||
: [255, 180, 0, 200], // orange when filtered out (no matching properties)
|
||||
getLineWidth: hasData ? 4 : 3,
|
||||
lineWidthUnits: 'pixels',
|
||||
stroked: true,
|
||||
filled: true,
|
||||
pickable: false,
|
||||
});
|
||||
}, [searchedPostcode, searchedPostcodeHasData]);
|
||||
|
||||
const layers = useMemo(() => {
|
||||
const baseLayers = usePostcodeView
|
||||
? [postcodeLayer, postcodeLabelsLayer, poiLayer]
|
||||
: [hexLayer, poiLayer];
|
||||
if (searchedPostcodeHighlightLayer) {
|
||||
return [...baseLayers, searchedPostcodeHighlightLayer];
|
||||
}
|
||||
return baseLayers;
|
||||
}, [
|
||||
const {
|
||||
layers,
|
||||
popupInfo,
|
||||
hoverPosition,
|
||||
countRange,
|
||||
postcodeCountRange,
|
||||
colorFeatureMeta,
|
||||
handleMouseLeave,
|
||||
} = useDeckLayers({
|
||||
data,
|
||||
postcodeData,
|
||||
usePostcodeView,
|
||||
hexLayer,
|
||||
postcodeLayer,
|
||||
postcodeLabelsLayer,
|
||||
poiLayer,
|
||||
searchedPostcodeHighlightLayer,
|
||||
]);
|
||||
|
||||
const handleMouseLeave = useCallback(() => {
|
||||
setHoverPosition(null);
|
||||
setHoveredPostcode(null);
|
||||
setPopupInfo(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}, []);
|
||||
pois,
|
||||
viewFeature,
|
||||
colorRange,
|
||||
filterRange,
|
||||
features,
|
||||
selectedHexagonId,
|
||||
hoveredHexagonId,
|
||||
onHexagonClick,
|
||||
onHexagonHover,
|
||||
theme,
|
||||
searchedPostcode,
|
||||
bounds: viewportBounds,
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="flex-1 h-full relative" ref={containerRef} onMouseLeave={handleMouseLeave}>
|
||||
|
|
|
|||
|
|
@ -201,7 +201,7 @@ export default function MapPage({
|
|||
.then((blob) => {
|
||||
const link = document.createElement('a');
|
||||
link.href = URL.createObjectURL(blob);
|
||||
link.download = 'narrowit-export.xlsx';
|
||||
link.download = 'perfect-postcodes-export.xlsx';
|
||||
link.click();
|
||||
URL.revokeObjectURL(link.href);
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { useState, useCallback } from 'react';
|
||||
import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups';
|
||||
import type { POICategoryGroup } from '../../types';
|
||||
import InfoPopup from '../ui/InfoPopup';
|
||||
import { SearchInput } from '../ui/SearchInput';
|
||||
|
|
@ -21,7 +22,7 @@ export default function POIPane({
|
|||
onNavigateToSource,
|
||||
}: POIPaneProps) {
|
||||
const [searchTerm, setSearchTerm] = useState('');
|
||||
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
|
||||
const [collapsedGroups, toggleCollapse] = useCollapsibleGroups();
|
||||
const [showInfo, setShowInfo] = useState(false);
|
||||
|
||||
const allCategories = groups.flatMap((g) => g.categories);
|
||||
|
|
@ -60,18 +61,6 @@ export default function POIPane({
|
|||
[groups, selectedCategories, onCategoriesChange]
|
||||
);
|
||||
|
||||
const toggleCollapse = (groupName: string) => {
|
||||
setCollapsedGroups((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(groupName)) {
|
||||
next.delete(groupName);
|
||||
} else {
|
||||
next.add(groupName);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const lowerSearch = searchTerm.toLowerCase();
|
||||
|
||||
const filteredGroups = groups
|
||||
|
|
|
|||
26
frontend/src/components/map/StreetViewEmbed.tsx
Normal file
26
frontend/src/components/map/StreetViewEmbed.tsx
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import type { HexagonLocation } from '../../lib/external-search';
|
||||
|
||||
interface StreetViewEmbedProps {
|
||||
location: HexagonLocation;
|
||||
}
|
||||
|
||||
export default function StreetViewEmbed({ location }: StreetViewEmbedProps) {
|
||||
return (
|
||||
<div>
|
||||
<div className="px-3 py-1.5 text-xs font-bold text-warm-500 bg-warm-50 dark:bg-warm-900 dark:text-warm-400 sticky top-0">
|
||||
Street View
|
||||
</div>
|
||||
<div className="px-3 py-2">
|
||||
<div className="rounded overflow-hidden border border-warm-200 dark:border-warm-700">
|
||||
<iframe
|
||||
className="w-full"
|
||||
style={{ height: 240, border: 0 }}
|
||||
loading="lazy"
|
||||
referrerPolicy="no-referrer-when-downgrade"
|
||||
src={`https://maps.google.com/maps?layer=c&cbll=${location.lat},${location.lon}&cbp=11,0,0,0,0&output=svembed`}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -5,12 +5,12 @@ import { BookmarkIcon } from './icons/BookmarkIcon';
|
|||
import { MapPinIcon } from './icons/MapPinIcon';
|
||||
import { CheckIcon } from './icons/CheckIcon';
|
||||
import { ClipboardIcon } from './icons/ClipboardIcon';
|
||||
import { CloseIcon } from './icons/CloseIcon';
|
||||
import { MenuIcon } from './icons/MenuIcon';
|
||||
import { SunIcon } from './icons/SunIcon';
|
||||
import { MoonIcon } from './icons/MoonIcon';
|
||||
import { SpinnerIcon } from './icons/SpinnerIcon';
|
||||
import UserMenu from './UserMenu';
|
||||
import MobileMenu from './MobileMenu';
|
||||
|
||||
export type Page = 'home' | 'dashboard' | 'data-sources' | 'faq' | 'saved-searches';
|
||||
|
||||
|
|
@ -89,23 +89,6 @@ export default function Header({
|
|||
: 'text-warm-300 hover:bg-navy-800 hover:text-white'
|
||||
}`;
|
||||
|
||||
const mobileNavItem = (page: Page, label: string) => (
|
||||
<button
|
||||
key={page}
|
||||
className={`w-full text-left px-4 py-3 text-base font-medium rounded ${
|
||||
activePage === page
|
||||
? 'bg-navy-700 text-white'
|
||||
: 'text-warm-300 hover:bg-navy-800 hover:text-white'
|
||||
}`}
|
||||
onClick={() => {
|
||||
onPageChange(page);
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
);
|
||||
|
||||
return (
|
||||
<header className="h-12 bg-navy-900 text-white flex items-center px-4 shrink-0">
|
||||
{/* Left: Logo + nav */}
|
||||
|
|
@ -115,7 +98,7 @@ export default function Header({
|
|||
onClick={() => onPageChange('home')}
|
||||
>
|
||||
<MapPinIcon className="w-5 h-5 text-teal-400" />
|
||||
<span className="font-semibold text-lg">Narrowit</span>
|
||||
<span className="font-semibold text-lg">Perfect Postcodes</span>
|
||||
</button>
|
||||
|
||||
{/* Desktop nav */}
|
||||
|
|
@ -251,134 +234,23 @@ export default function Header({
|
|||
|
||||
{/* Mobile slide-in menu */}
|
||||
{isMobile && menuOpen && (
|
||||
<>
|
||||
{/* Backdrop */}
|
||||
<div className="fixed inset-0 bg-black/50 z-40" onClick={() => setMenuOpen(false)} />
|
||||
{/* Menu panel */}
|
||||
<div className="fixed top-0 right-0 bottom-0 w-64 bg-navy-900 z-50 flex flex-col shadow-xl">
|
||||
<div className="flex items-center justify-between px-4 h-12 border-b border-navy-700">
|
||||
<span className="font-semibold">Menu</span>
|
||||
<button
|
||||
onClick={() => setMenuOpen(false)}
|
||||
className="flex items-center justify-center w-10 h-10 -mr-2 rounded hover:bg-navy-800 transition-colors"
|
||||
aria-label="Close menu"
|
||||
>
|
||||
<CloseIcon className="w-5 h-5" />
|
||||
</button>
|
||||
</div>
|
||||
<nav className="flex-1 flex flex-col gap-1 p-3 overflow-y-auto">
|
||||
{mobileNavItem('dashboard', 'Dashboard')}
|
||||
{user && mobileNavItem('saved-searches', 'Saved')}
|
||||
{mobileNavItem('data-sources', 'Data Sources')}
|
||||
{mobileNavItem('faq', 'FAQ')}
|
||||
|
||||
{/* Dashboard actions */}
|
||||
{activePage === 'dashboard' && (
|
||||
<div className="mt-3 pt-3 border-t border-navy-700 flex flex-col gap-1">
|
||||
{onSaveSearch && (
|
||||
<button
|
||||
onClick={() => {
|
||||
onSaveSearch();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
disabled={savingSearch}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded disabled:opacity-50"
|
||||
>
|
||||
{savingSearch ? (
|
||||
<SpinnerIcon className="w-5 h-5 animate-spin" />
|
||||
) : (
|
||||
<BookmarkIcon className="w-5 h-5" />
|
||||
)}
|
||||
Save
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={() => {
|
||||
handleShare();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded"
|
||||
>
|
||||
{copied ? (
|
||||
<CheckIcon className="w-5 h-5" />
|
||||
) : (
|
||||
<ClipboardIcon className="w-5 h-5" />
|
||||
)}
|
||||
{copied ? 'Copied!' : 'Share'}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
onExport?.();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
disabled={!onExport || exporting}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded disabled:opacity-50"
|
||||
>
|
||||
<DownloadIcon className="w-5 h-5" />
|
||||
{exporting ? 'Exporting...' : 'Export'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</nav>
|
||||
|
||||
{/* Theme toggle + Auth section at bottom */}
|
||||
<div className="p-3 border-t border-navy-700 flex flex-col gap-3">
|
||||
{/* Theme toggle */}
|
||||
<button
|
||||
onClick={() => {
|
||||
onToggleTheme();
|
||||
}}
|
||||
className="w-full flex items-center gap-3 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded transition-colors"
|
||||
>
|
||||
{theme === 'light' ? (
|
||||
<SunIcon className="w-5 h-5" />
|
||||
) : (
|
||||
<MoonIcon className="w-5 h-5" />
|
||||
)}
|
||||
<span>Theme: {theme === 'light' ? 'Light' : 'Dark'}</span>
|
||||
</button>
|
||||
|
||||
{/* Auth buttons */}
|
||||
<div>
|
||||
{user ? (
|
||||
<div className="flex items-center justify-between px-4 py-2">
|
||||
<span className="text-sm text-warm-300 truncate">{user.email}</span>
|
||||
<button
|
||||
onClick={() => {
|
||||
onLogout();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
className="text-sm text-warm-400 hover:text-white"
|
||||
>
|
||||
Log out
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => {
|
||||
onLoginClick();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
className="flex-1 px-3 py-2.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm text-center"
|
||||
>
|
||||
Log in
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
onRegisterClick();
|
||||
setMenuOpen(false);
|
||||
}}
|
||||
className="flex-1 px-3 py-2.5 rounded bg-teal-600 hover:bg-teal-700 transition-colors text-sm font-medium text-center"
|
||||
>
|
||||
Register
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
<MobileMenu
|
||||
activePage={activePage}
|
||||
onPageChange={onPageChange}
|
||||
theme={theme}
|
||||
onToggleTheme={onToggleTheme}
|
||||
onExport={onExport}
|
||||
exporting={exporting}
|
||||
onSaveSearch={onSaveSearch}
|
||||
savingSearch={savingSearch}
|
||||
user={user}
|
||||
onLoginClick={onLoginClick}
|
||||
onRegisterClick={onRegisterClick}
|
||||
onLogout={onLogout}
|
||||
onClose={() => setMenuOpen(false)}
|
||||
onShare={handleShare}
|
||||
copied={copied}
|
||||
/>
|
||||
)}
|
||||
</header>
|
||||
);
|
||||
|
|
|
|||
186
frontend/src/components/ui/MobileMenu.tsx
Normal file
186
frontend/src/components/ui/MobileMenu.tsx
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
import type { Page } from './Header';
|
||||
import type { AuthUser } from '../../hooks/useAuth';
|
||||
import { DownloadIcon } from './icons/DownloadIcon';
|
||||
import { BookmarkIcon } from './icons/BookmarkIcon';
|
||||
import { CheckIcon } from './icons/CheckIcon';
|
||||
import { ClipboardIcon } from './icons/ClipboardIcon';
|
||||
import { CloseIcon } from './icons/CloseIcon';
|
||||
import { SunIcon } from './icons/SunIcon';
|
||||
import { MoonIcon } from './icons/MoonIcon';
|
||||
import { SpinnerIcon } from './icons/SpinnerIcon';
|
||||
|
||||
interface MobileMenuProps {
|
||||
activePage: Page;
|
||||
onPageChange: (page: Page) => void;
|
||||
theme: 'light' | 'dark';
|
||||
onToggleTheme: () => void;
|
||||
onExport: (() => void) | null;
|
||||
exporting: boolean;
|
||||
onSaveSearch: (() => void) | null;
|
||||
savingSearch: boolean;
|
||||
user: AuthUser | null;
|
||||
onLoginClick: () => void;
|
||||
onRegisterClick: () => void;
|
||||
onLogout: () => void;
|
||||
onClose: () => void;
|
||||
onShare: () => void;
|
||||
copied: boolean;
|
||||
}
|
||||
|
||||
export default function MobileMenu({
|
||||
activePage,
|
||||
onPageChange,
|
||||
theme,
|
||||
onToggleTheme,
|
||||
onExport,
|
||||
exporting,
|
||||
onSaveSearch,
|
||||
savingSearch,
|
||||
user,
|
||||
onLoginClick,
|
||||
onRegisterClick,
|
||||
onLogout,
|
||||
onClose,
|
||||
onShare,
|
||||
copied,
|
||||
}: MobileMenuProps) {
|
||||
const mobileNavItem = (page: Page, label: string) => (
|
||||
<button
|
||||
key={page}
|
||||
className={`w-full text-left px-4 py-3 text-base font-medium rounded ${
|
||||
activePage === page
|
||||
? 'bg-navy-700 text-white'
|
||||
: 'text-warm-300 hover:bg-navy-800 hover:text-white'
|
||||
}`}
|
||||
onClick={() => {
|
||||
onPageChange(page);
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
{/* Backdrop */}
|
||||
<div className="fixed inset-0 bg-black/50 z-40" onClick={onClose} />
|
||||
{/* Menu panel */}
|
||||
<div className="fixed top-0 right-0 bottom-0 w-64 bg-navy-900 z-50 flex flex-col shadow-xl">
|
||||
<div className="flex items-center justify-between px-4 h-12 border-b border-navy-700">
|
||||
<span className="font-semibold">Menu</span>
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="flex items-center justify-center w-10 h-10 -mr-2 rounded hover:bg-navy-800 transition-colors"
|
||||
aria-label="Close menu"
|
||||
>
|
||||
<CloseIcon className="w-5 h-5" />
|
||||
</button>
|
||||
</div>
|
||||
<nav className="flex-1 flex flex-col gap-1 p-3 overflow-y-auto">
|
||||
{mobileNavItem('dashboard', 'Dashboard')}
|
||||
{user && mobileNavItem('saved-searches', 'Saved')}
|
||||
{mobileNavItem('data-sources', 'Data Sources')}
|
||||
{mobileNavItem('faq', 'FAQ')}
|
||||
|
||||
{/* Dashboard actions */}
|
||||
{activePage === 'dashboard' && (
|
||||
<div className="mt-3 pt-3 border-t border-navy-700 flex flex-col gap-1">
|
||||
{onSaveSearch && (
|
||||
<button
|
||||
onClick={() => {
|
||||
onSaveSearch();
|
||||
onClose();
|
||||
}}
|
||||
disabled={savingSearch}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded disabled:opacity-50"
|
||||
>
|
||||
{savingSearch ? (
|
||||
<SpinnerIcon className="w-5 h-5 animate-spin" />
|
||||
) : (
|
||||
<BookmarkIcon className="w-5 h-5" />
|
||||
)}
|
||||
Save
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={() => {
|
||||
onShare();
|
||||
onClose();
|
||||
}}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded"
|
||||
>
|
||||
{copied ? <CheckIcon className="w-5 h-5" /> : <ClipboardIcon className="w-5 h-5" />}
|
||||
{copied ? 'Copied!' : 'Share'}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
onExport?.();
|
||||
onClose();
|
||||
}}
|
||||
disabled={!onExport || exporting}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded disabled:opacity-50"
|
||||
>
|
||||
<DownloadIcon className="w-5 h-5" />
|
||||
{exporting ? 'Exporting...' : 'Export'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</nav>
|
||||
|
||||
{/* Theme toggle + Auth section at bottom */}
|
||||
<div className="p-3 border-t border-navy-700 flex flex-col gap-3">
|
||||
{/* Theme toggle */}
|
||||
<button
|
||||
onClick={() => {
|
||||
onToggleTheme();
|
||||
}}
|
||||
className="w-full flex items-center gap-3 px-4 py-3 text-base text-warm-300 hover:bg-navy-800 hover:text-white rounded transition-colors"
|
||||
>
|
||||
{theme === 'light' ? <SunIcon className="w-5 h-5" /> : <MoonIcon className="w-5 h-5" />}
|
||||
<span>Theme: {theme === 'light' ? 'Light' : 'Dark'}</span>
|
||||
</button>
|
||||
|
||||
{/* Auth buttons */}
|
||||
<div>
|
||||
{user ? (
|
||||
<div className="flex items-center justify-between px-4 py-2">
|
||||
<span className="text-sm text-warm-300 truncate">{user.email}</span>
|
||||
<button
|
||||
onClick={() => {
|
||||
onLogout();
|
||||
onClose();
|
||||
}}
|
||||
className="text-sm text-warm-400 hover:text-white"
|
||||
>
|
||||
Log out
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => {
|
||||
onLoginClick();
|
||||
onClose();
|
||||
}}
|
||||
className="flex-1 px-3 py-2.5 rounded bg-navy-800 hover:bg-navy-700 transition-colors text-sm text-center"
|
||||
>
|
||||
Log in
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
onRegisterClick();
|
||||
onClose();
|
||||
}}
|
||||
className="flex-1 px-3 py-2.5 rounded bg-teal-600 hover:bg-teal-700 transition-colors text-sm font-medium text-center"
|
||||
>
|
||||
Register
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
16
frontend/src/hooks/useCollapsibleGroups.ts
Normal file
16
frontend/src/hooks/useCollapsibleGroups.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import { useState, useCallback } from 'react';
|
||||
|
||||
export function useCollapsibleGroups(): [Set<string>, (name: string) => void] {
|
||||
const [collapsed, setCollapsed] = useState<Set<string>>(new Set());
|
||||
|
||||
const toggle = useCallback((name: string) => {
|
||||
setCollapsed((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(name)) next.delete(name);
|
||||
else next.add(name);
|
||||
return next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
return [collapsed, toggle];
|
||||
}
|
||||
495
frontend/src/hooks/useDeckLayers.ts
Normal file
495
frontend/src/hooks/useDeckLayers.ts
Normal file
|
|
@ -0,0 +1,495 @@
|
|||
import { useCallback, useRef, useState, useMemo } from 'react';
|
||||
import { H3HexagonLayer } from '@deck.gl/geo-layers';
|
||||
import { GeoJsonLayer, IconLayer, TextLayer } from '@deck.gl/layers';
|
||||
import type { PickingInfo } from '@deck.gl/core';
|
||||
import type {
|
||||
HexagonData,
|
||||
PostcodeFeature,
|
||||
PostcodeProperties,
|
||||
POI,
|
||||
FeatureMeta,
|
||||
Bounds,
|
||||
} from '../types';
|
||||
import type { SearchedPostcode } from '../components/map/PostcodeSearch';
|
||||
import {
|
||||
emojiToTwemojiUrl,
|
||||
DENSITY_GRADIENT,
|
||||
DENSITY_GRADIENT_DARK,
|
||||
getFeatureFillColor,
|
||||
} from '../lib/map-utils';
|
||||
|
||||
/** Convert POI id (e.g. "n12345") to OpenStreetMap URL */
|
||||
function osmIdToUrl(id: string): string | null {
|
||||
const match = id.match(/^([nwr])(\d+)$/);
|
||||
if (!match) return null;
|
||||
const typeMap: Record<string, string> = { n: 'node', w: 'way', r: 'relation' };
|
||||
return `https://www.openstreetmap.org/${typeMap[match[1]]}/${match[2]}`;
|
||||
}
|
||||
|
||||
export { osmIdToUrl };
|
||||
|
||||
interface UseDeckLayersProps {
|
||||
data: HexagonData[];
|
||||
postcodeData: PostcodeFeature[];
|
||||
usePostcodeView: boolean;
|
||||
pois: POI[];
|
||||
viewFeature: string | null;
|
||||
colorRange: [number, number] | null;
|
||||
filterRange: [number, number] | null;
|
||||
features: FeatureMeta[];
|
||||
selectedHexagonId: string | null;
|
||||
hoveredHexagonId: string | null;
|
||||
onHexagonClick: (id: string, isPostcode?: boolean) => void;
|
||||
onHexagonHover: (h3: string | null, x?: number, y?: number) => void;
|
||||
theme: 'light' | 'dark';
|
||||
searchedPostcode?: SearchedPostcode | null;
|
||||
bounds?: Bounds | null;
|
||||
}
|
||||
|
||||
export interface PopupInfo {
|
||||
x: number;
|
||||
y: number;
|
||||
name: string;
|
||||
category: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export function useDeckLayers({
|
||||
data,
|
||||
postcodeData,
|
||||
usePostcodeView,
|
||||
pois,
|
||||
viewFeature,
|
||||
colorRange,
|
||||
filterRange,
|
||||
features,
|
||||
selectedHexagonId,
|
||||
hoveredHexagonId,
|
||||
onHexagonClick,
|
||||
onHexagonHover,
|
||||
theme,
|
||||
searchedPostcode,
|
||||
bounds: viewportBounds,
|
||||
}: UseDeckLayersProps) {
|
||||
const [popupInfo, setPopupInfo] = useState<PopupInfo | null>(null);
|
||||
const [hoverPosition, setHoverPosition] = useState<{ x: number; y: number } | null>(null);
|
||||
const [selectedPostcode, setSelectedPostcode] = useState<string | null>(null);
|
||||
const [hoveredPostcode, setHoveredPostcode] = useState<string | null>(null);
|
||||
|
||||
const isDark = theme === 'dark';
|
||||
const densityGradient = isDark ? DENSITY_GRADIENT_DARK : DENSITY_GRADIENT;
|
||||
|
||||
// --- Refs for deck.gl accessors (avoid re-creating layers on every change) ---
|
||||
const viewFeatureRef = useRef(viewFeature);
|
||||
viewFeatureRef.current = viewFeature;
|
||||
const colorRangeRef = useRef(colorRange);
|
||||
colorRangeRef.current = colorRange;
|
||||
const filterRangeRef = useRef(filterRange);
|
||||
filterRangeRef.current = filterRange;
|
||||
const isDarkRef = useRef(isDark);
|
||||
isDarkRef.current = isDark;
|
||||
const densityGradientRef = useRef(densityGradient);
|
||||
densityGradientRef.current = densityGradient;
|
||||
const selectedHexagonIdRef = useRef(selectedHexagonId);
|
||||
selectedHexagonIdRef.current = selectedHexagonId;
|
||||
const hoveredHexagonIdRef = useRef(hoveredHexagonId);
|
||||
hoveredHexagonIdRef.current = hoveredHexagonId;
|
||||
const selectedPostcodeRef = useRef(selectedPostcode);
|
||||
selectedPostcodeRef.current = selectedPostcode;
|
||||
const hoveredPostcodeRef = useRef(hoveredPostcode);
|
||||
hoveredPostcodeRef.current = hoveredPostcode;
|
||||
|
||||
const colorFeatureMeta = useMemo(
|
||||
() => (viewFeature ? features.find((f) => f.name === viewFeature) || null : null),
|
||||
[viewFeature, features]
|
||||
);
|
||||
const colorFeatureMetaRef = useRef(colorFeatureMeta);
|
||||
colorFeatureMetaRef.current = colorFeatureMeta;
|
||||
|
||||
// --- Count ranges ---
|
||||
const countRange = useMemo(() => {
|
||||
if (data.length === 0) return { min: 0, max: 1 };
|
||||
let min = Infinity;
|
||||
let max = -Infinity;
|
||||
for (const d of data) {
|
||||
if (viewportBounds) {
|
||||
if (
|
||||
d.lat < viewportBounds.south ||
|
||||
d.lat > viewportBounds.north ||
|
||||
d.lon < viewportBounds.west ||
|
||||
d.lon > viewportBounds.east
|
||||
)
|
||||
continue;
|
||||
}
|
||||
const c = d.count as number;
|
||||
if (c < min) min = c;
|
||||
if (c > max) max = c;
|
||||
}
|
||||
if (min === Infinity) return { min: 0, max: 1 };
|
||||
if (min === max) return { min, max: min + 1 };
|
||||
return { min, max };
|
||||
}, [data, viewportBounds]);
|
||||
|
||||
const countRangeRef = useRef(countRange);
|
||||
countRangeRef.current = countRange;
|
||||
|
||||
const postcodeCountRange = useMemo(() => {
|
||||
if (postcodeData.length === 0) return { min: 0, max: 1 };
|
||||
let min = Infinity;
|
||||
let max = -Infinity;
|
||||
for (const d of postcodeData) {
|
||||
if (viewportBounds) {
|
||||
const [lng, lat] = d.properties.centroid as [number, number];
|
||||
if (
|
||||
lat < viewportBounds.south ||
|
||||
lat > viewportBounds.north ||
|
||||
lng < viewportBounds.west ||
|
||||
lng > viewportBounds.east
|
||||
)
|
||||
continue;
|
||||
}
|
||||
const c = d.properties.count;
|
||||
if (c < min) min = c;
|
||||
if (c > max) max = c;
|
||||
}
|
||||
if (min === Infinity) return { min: 0, max: 1 };
|
||||
if (min === max) return { min, max: min + 1 };
|
||||
return { min, max };
|
||||
}, [postcodeData, viewportBounds]);
|
||||
|
||||
const postcodeCountRangeRef = useRef(postcodeCountRange);
|
||||
postcodeCountRangeRef.current = postcodeCountRange;
|
||||
|
||||
// --- Click/hover handlers ---
|
||||
const onHexagonClickRef = useRef(onHexagonClick);
|
||||
onHexagonClickRef.current = onHexagonClick;
|
||||
const handleHexagonClick = useCallback((info: PickingInfo<HexagonData>) => {
|
||||
if (info.object && 'h3' in info.object) {
|
||||
onHexagonClickRef.current(info.object.h3);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const onHexagonHoverRef = useRef(onHexagonHover);
|
||||
onHexagonHoverRef.current = onHexagonHover;
|
||||
const handleHexagonHover = useCallback((info: PickingInfo<HexagonData>) => {
|
||||
if (info.object && 'h3' in info.object && info.x !== undefined && info.y !== undefined) {
|
||||
setHoverPosition({ x: info.x, y: info.y });
|
||||
onHexagonHoverRef.current(info.object.h3, info.x, info.y);
|
||||
} else {
|
||||
setHoverPosition(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handlePoiHover = useCallback((info: PickingInfo<POI>) => {
|
||||
if (info.object && info.x !== undefined && info.y !== undefined) {
|
||||
setPopupInfo({
|
||||
x: info.x,
|
||||
y: info.y,
|
||||
name: info.object.name,
|
||||
category: info.object.category,
|
||||
id: info.object.id,
|
||||
});
|
||||
} else {
|
||||
setPopupInfo(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handlePoiHoverRef = useRef(handlePoiHover);
|
||||
handlePoiHoverRef.current = handlePoiHover;
|
||||
const stablePoiHover = useCallback((info: PickingInfo<POI>) => {
|
||||
handlePoiHoverRef.current(info);
|
||||
}, []);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handlePostcodeClick = useCallback((info: PickingInfo<any>) => {
|
||||
const pc = info.object?.properties?.postcode;
|
||||
if (pc) {
|
||||
setSelectedPostcode((prev) => (prev === pc ? null : pc));
|
||||
onHexagonClickRef.current(pc, true);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handlePostcodeHoverCallback = useCallback((info: PickingInfo<any>) => {
|
||||
const pc = info.object?.properties?.postcode;
|
||||
if (pc && info.x !== undefined && info.y !== undefined) {
|
||||
setHoveredPostcode(pc);
|
||||
setHoverPosition({ x: info.x, y: info.y });
|
||||
onHexagonHoverRef.current(pc, info.x, info.y);
|
||||
} else {
|
||||
setHoveredPostcode(null);
|
||||
setHoverPosition(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// --- Color triggers ---
|
||||
const colorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${countRange.min}|${countRange.max}|${selectedHexagonId}|${hoveredHexagonId}|${theme}`;
|
||||
const postcodeColorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${postcodeCountRange.min}|${postcodeCountRange.max}|${selectedPostcode}|${hoveredPostcode}|${theme}`;
|
||||
|
||||
// --- Layers ---
|
||||
const hexLayer = useMemo(
|
||||
() =>
|
||||
new H3HexagonLayer<HexagonData>({
|
||||
id: 'h3-hexagons',
|
||||
data,
|
||||
getHexagon: (d) => d.h3,
|
||||
getFillColor: (d) => {
|
||||
const vf = viewFeatureRef.current;
|
||||
const clr = colorRangeRef.current;
|
||||
const fr = filterRangeRef.current;
|
||||
const cfm = colorFeatureMetaRef.current;
|
||||
const dark = isDarkRef.current;
|
||||
if (vf && clr && cfm) {
|
||||
const val = d[`avg_${vf}`] ?? d[`min_${vf}`];
|
||||
const minVal = d[`min_${vf}`] as number | undefined;
|
||||
const maxVal = d[`max_${vf}`] as number | undefined;
|
||||
return getFeatureFillColor(
|
||||
val as number | null | undefined,
|
||||
minVal,
|
||||
maxVal,
|
||||
clr,
|
||||
fr,
|
||||
0,
|
||||
densityGradientRef.current,
|
||||
dark,
|
||||
255
|
||||
);
|
||||
}
|
||||
const cr = countRangeRef.current;
|
||||
const c = d.count as number;
|
||||
const t = (c - cr.min) / (cr.max - cr.min);
|
||||
return getFeatureFillColor(
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
null,
|
||||
null,
|
||||
t,
|
||||
densityGradientRef.current,
|
||||
dark,
|
||||
255
|
||||
);
|
||||
},
|
||||
getLineColor: (d) => {
|
||||
if (d.h3 === selectedHexagonIdRef.current)
|
||||
return [255, 255, 255, 255] as [number, number, number, number];
|
||||
if (d.h3 === hoveredHexagonIdRef.current)
|
||||
return [29, 228, 195, 200] as [number, number, number, number];
|
||||
return [0, 0, 0, 0] as [number, number, number, number];
|
||||
},
|
||||
getLineWidth: (d) => {
|
||||
if (d.h3 === selectedHexagonIdRef.current) return 3;
|
||||
if (d.h3 === hoveredHexagonIdRef.current) return 2;
|
||||
return 0;
|
||||
},
|
||||
lineWidthUnits: 'pixels',
|
||||
updateTriggers: {
|
||||
getFillColor: [colorTrigger],
|
||||
getLineColor: [colorTrigger],
|
||||
getLineWidth: [colorTrigger],
|
||||
},
|
||||
extruded: false,
|
||||
pickable: true,
|
||||
opacity: 1,
|
||||
highPrecision: true,
|
||||
onClick: handleHexagonClick,
|
||||
onHover: handleHexagonHover,
|
||||
// @ts-expect-error beforeId is a MapboxOverlay interleave prop, not typed in LayerProps
|
||||
beforeId: 'landuse_park',
|
||||
}),
|
||||
[data, colorTrigger, handleHexagonClick, handleHexagonHover]
|
||||
);
|
||||
|
||||
const postcodeLayer = useMemo(
|
||||
() =>
|
||||
new GeoJsonLayer<PostcodeProperties>({
|
||||
id: 'postcode-polygons',
|
||||
data: postcodeData as PostcodeFeature[],
|
||||
getFillColor: (f) => {
|
||||
const d = f.properties;
|
||||
const vf = viewFeatureRef.current;
|
||||
const clr = colorRangeRef.current;
|
||||
const fr = filterRangeRef.current;
|
||||
const cfm = colorFeatureMetaRef.current;
|
||||
const dark = isDarkRef.current;
|
||||
if (vf && clr && cfm) {
|
||||
const val = d[`avg_${vf}`] ?? d[`min_${vf}`];
|
||||
const minVal = d[`min_${vf}`] as number | undefined;
|
||||
const maxVal = d[`max_${vf}`] as number | undefined;
|
||||
return getFeatureFillColor(
|
||||
val as number | null | undefined,
|
||||
minVal,
|
||||
maxVal,
|
||||
clr,
|
||||
fr,
|
||||
0,
|
||||
densityGradientRef.current,
|
||||
dark,
|
||||
180
|
||||
);
|
||||
}
|
||||
const cr = postcodeCountRangeRef.current;
|
||||
const c = d.count;
|
||||
const t = (c - cr.min) / (cr.max - cr.min);
|
||||
return getFeatureFillColor(
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
null,
|
||||
null,
|
||||
t,
|
||||
densityGradientRef.current,
|
||||
dark,
|
||||
180
|
||||
);
|
||||
},
|
||||
getLineColor: (f) => {
|
||||
const pc = f.properties.postcode;
|
||||
const dark = isDarkRef.current;
|
||||
if (pc === selectedPostcodeRef.current)
|
||||
return [255, 255, 255, 255] as [number, number, number, number];
|
||||
if (pc === hoveredPostcodeRef.current)
|
||||
return [29, 228, 195, 200] as [number, number, number, number];
|
||||
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
},
|
||||
getLineWidth: (f) => {
|
||||
const pc = f.properties.postcode;
|
||||
if (pc === selectedPostcodeRef.current) return 3;
|
||||
if (pc === hoveredPostcodeRef.current) return 2;
|
||||
return 1;
|
||||
},
|
||||
lineWidthUnits: 'pixels',
|
||||
updateTriggers: {
|
||||
getFillColor: [postcodeColorTrigger],
|
||||
getLineColor: [postcodeColorTrigger],
|
||||
getLineWidth: [postcodeColorTrigger],
|
||||
},
|
||||
extruded: false,
|
||||
pickable: true,
|
||||
onClick: handlePostcodeClick,
|
||||
onHover: handlePostcodeHoverCallback,
|
||||
// @ts-expect-error beforeId is a MapboxOverlay interleave prop, not typed in LayerProps
|
||||
beforeId: 'landuse_park',
|
||||
}),
|
||||
[postcodeData, postcodeColorTrigger, handlePostcodeClick, handlePostcodeHoverCallback]
|
||||
);
|
||||
|
||||
const postcodeLabelsLayer = useMemo(
|
||||
() =>
|
||||
new TextLayer<PostcodeFeature>({
|
||||
id: 'postcode-labels',
|
||||
data: postcodeData,
|
||||
getPosition: (f) => f.properties.centroid,
|
||||
getText: (f) => f.properties.postcode,
|
||||
getSize: 12,
|
||||
getColor: theme === 'dark' ? [255, 255, 255, 240] : [40, 40, 40, 220],
|
||||
getTextAnchor: 'middle',
|
||||
getAlignmentBaseline: 'center',
|
||||
fontFamily: 'Inter, system-ui, sans-serif',
|
||||
fontWeight: 600,
|
||||
outlineWidth: 2,
|
||||
outlineColor: theme === 'dark' ? [30, 30, 30, 200] : [255, 255, 255, 200],
|
||||
sizeUnits: 'pixels',
|
||||
sizeMinPixels: 10,
|
||||
sizeMaxPixels: 14,
|
||||
billboard: false,
|
||||
pickable: false,
|
||||
}),
|
||||
[postcodeData, theme]
|
||||
);
|
||||
|
||||
const poiLayer = useMemo(
|
||||
() =>
|
||||
new IconLayer<POI>({
|
||||
id: 'poi-icons',
|
||||
data: pois,
|
||||
getPosition: (d) => [d.lng, d.lat],
|
||||
getIcon: (d) => ({
|
||||
url: emojiToTwemojiUrl(d.emoji),
|
||||
width: 72,
|
||||
height: 72,
|
||||
}),
|
||||
getSize: 24,
|
||||
sizeMinPixels: 20,
|
||||
sizeMaxPixels: 40,
|
||||
pickable: true,
|
||||
onHover: stablePoiHover,
|
||||
}),
|
||||
[pois, stablePoiHover]
|
||||
);
|
||||
|
||||
// Check if the searched postcode has data (passes current filters)
|
||||
const searchedPostcodeHasData = useMemo(() => {
|
||||
if (!searchedPostcode) return false;
|
||||
return postcodeData.some((f) => f.properties.postcode === searchedPostcode.postcode);
|
||||
}, [searchedPostcode, postcodeData]);
|
||||
|
||||
// Highlight layer for searched postcode
|
||||
const searchedPostcodeHighlightLayer = useMemo(() => {
|
||||
if (!searchedPostcode) return null;
|
||||
const hasData = searchedPostcodeHasData;
|
||||
const feature = {
|
||||
type: 'Feature' as const,
|
||||
geometry: searchedPostcode.geometry,
|
||||
properties: {},
|
||||
};
|
||||
return new GeoJsonLayer({
|
||||
id: 'searched-postcode-highlight',
|
||||
data: [feature],
|
||||
getFillColor: hasData
|
||||
? [29, 228, 195, 40] // teal tint when has data
|
||||
: [255, 180, 0, 30], // orange tint when filtered out
|
||||
getLineColor: hasData
|
||||
? [29, 228, 195, 255] // solid teal when has data
|
||||
: [255, 180, 0, 200], // orange when filtered out (no matching properties)
|
||||
getLineWidth: hasData ? 4 : 3,
|
||||
lineWidthUnits: 'pixels',
|
||||
stroked: true,
|
||||
filled: true,
|
||||
pickable: false,
|
||||
});
|
||||
}, [searchedPostcode, searchedPostcodeHasData]);
|
||||
|
||||
const layers = useMemo(() => {
|
||||
const baseLayers = usePostcodeView
|
||||
? [postcodeLayer, postcodeLabelsLayer, poiLayer]
|
||||
: [hexLayer, poiLayer];
|
||||
if (searchedPostcodeHighlightLayer) {
|
||||
return [...baseLayers, searchedPostcodeHighlightLayer];
|
||||
}
|
||||
return baseLayers;
|
||||
}, [
|
||||
usePostcodeView,
|
||||
hexLayer,
|
||||
postcodeLayer,
|
||||
postcodeLabelsLayer,
|
||||
poiLayer,
|
||||
searchedPostcodeHighlightLayer,
|
||||
]);
|
||||
|
||||
const handleMouseLeave = useCallback(() => {
|
||||
setHoverPosition(null);
|
||||
setHoveredPostcode(null);
|
||||
setPopupInfo(null);
|
||||
onHexagonHoverRef.current(null);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
layers,
|
||||
popupInfo,
|
||||
hoverPosition,
|
||||
countRange,
|
||||
postcodeCountRange,
|
||||
colorFeatureMeta,
|
||||
handleMouseLeave,
|
||||
selectedPostcode,
|
||||
hoveredPostcode,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { init as plausibleInit } from '@plausible-analytics/tracker';
|
||||
|
||||
plausibleInit({
|
||||
domain: 'narrowit.schmelczer.dev',
|
||||
domain: 'perfect-postcode.co.uk',
|
||||
endpoint: 'https://stats.schmelczer.dev/status',
|
||||
autoCapturePageviews: true,
|
||||
captureOnLocalhost: true,
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Narrowit — Every neighbourhood in England & Wales</title>
|
||||
<title>Perfect Postcodes — Every neighbourhood in England & Wales</title>
|
||||
<meta name="description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />
|
||||
<meta name="x-og-placeholder" content="__NARROWIT_OG_TAGS__" />
|
||||
<meta name="x-og-placeholder" content="__PERFECT_POSTCODES_OG_TAGS__" />
|
||||
<script>
|
||||
(function() {
|
||||
if (localStorage.getItem('theme') === 'dark') {
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ export function getMapStyle(theme: 'light' | 'dark'): StyleSpecification {
|
|||
} as StyleSpecification;
|
||||
}
|
||||
|
||||
type GradientStop = { t: number; color: [number, number, number] };
|
||||
export type GradientStop = { t: number; color: [number, number, number] };
|
||||
|
||||
// Oklab color space for perceptually uniform interpolation
|
||||
function srgbToLinear(c: number): number {
|
||||
|
|
@ -200,3 +200,47 @@ export function emojiToTwemojiUrl(emoji: string): string {
|
|||
const hex = codePoint.toString(16);
|
||||
return `${TWEMOJI_BASE}${hex}.png`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared fill-color logic for hex and postcode layers.
|
||||
* When a viewFeature is active, normalizes by colorRange and applies the feature gradient.
|
||||
* Otherwise falls back to density-based coloring using countRange.
|
||||
*/
|
||||
export function getFeatureFillColor(
|
||||
value: number | null | undefined,
|
||||
minVal: number | undefined,
|
||||
maxVal: number | undefined,
|
||||
colorRange: [number, number] | null,
|
||||
filterRange: [number, number] | null,
|
||||
countNormalized: number,
|
||||
densityGradient: GradientStop[],
|
||||
isDark: boolean,
|
||||
alpha: number
|
||||
): [number, number, number, number] {
|
||||
if (colorRange) {
|
||||
if (value == null)
|
||||
return (isDark ? [80, 70, 65, 80] : [128, 128, 128, 80]) as [number, number, number, number];
|
||||
if (filterRange) {
|
||||
if ((maxVal as number) < filterRange[0] || (minVal as number) > filterRange[1]) {
|
||||
return (isDark ? [60, 55, 50, 60] : [180, 180, 180, 60]) as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
}
|
||||
const range = colorRange[1] - colorRange[0];
|
||||
if (range === 0)
|
||||
return [...FEATURE_GRADIENT[0].color, alpha] as [number, number, number, number];
|
||||
const t = ((value as number) - colorRange[0]) / range;
|
||||
const rgb = normalizedToColor(Math.max(0, Math.min(1, t)));
|
||||
return [...rgb, alpha] as [number, number, number, number];
|
||||
}
|
||||
return [...countToColor(Math.max(0, Math.min(1, countNormalized)), densityGradient), alpha] as [
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
number,
|
||||
];
|
||||
}
|
||||
|
|
|
|||
21
homepage.md
Normal file
21
homepage.md
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
(above title) Browsing listings is not a strategy. Knowing what you want is.
|
||||
|
||||
(title) Find your <green>perfect postcode</green> before you find your property.
|
||||
|
||||
Set the sliders according to your expectations and the map will highlight the areas that actually match. Instantly.
|
||||
|
||||
<example with two sliders, last known price, minor crime/>
|
||||
|
||||
That's just two. We've built 43 — spanning transport links, amenities, demographics, environment risk, broadband speeds, crime, and more. (show the filter types with small cards)
|
||||
|
||||
Here's the problem with property search: listings only show you what's on the market right now — a thin slice of what an area is actually like. And even if you could look beyond them, there are millions of postcodes across England. You can't research them all yourself.
|
||||
|
||||
So we've built this dashboard for You. We took millions of historical transactions and public records, connected and extended them with proprietary algorithms on top — so the map doesn't just show you raw data, it surfaces the patterns that matter.
|
||||
|
||||
Understand areas first. Then find the right property within them, with expectations you've set yourself rather than ones the market set for you.
|
||||
|
||||
(Fun cereal graphic on the side with this popup) You might buy a box of cereal because it's 20% off. Your next home is not a box of cereal. Don't let a discount on the wrong property distract you from finding the right one. Know what you're looking for, then go looking.
|
||||
|
||||
The biggest financial decision of your life deserves proper tools behind it.
|
||||
|
||||
[Explore the map] Button
|
||||
354
notebooks/price_model_evaluation.ipynb
Normal file
354
notebooks/price_model_evaluation.ipynb
Normal file
|
|
@ -0,0 +1,354 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Price Estimation Model Evaluation\n",
|
||||
"\n",
|
||||
"Evaluates the repeat-sales price index model that adjusts each property's\n",
|
||||
"last known sale price to the current year using a hierarchical postcode-sector index."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import polars as pl\n",
|
||||
"import numpy as np\n",
|
||||
"import plotly.express as px\n",
|
||||
"import plotly.graph_objects as go\n",
|
||||
"from plotly.subplots import make_subplots\n",
|
||||
"\n",
|
||||
"pl.Config.set_tbl_rows(20)\n",
|
||||
"pl.Config.set_fmt_str_lengths(50)\n",
|
||||
"\n",
|
||||
"DATA_DIR = \"../property-data\"\n",
|
||||
"\n",
|
||||
"index_df = pl.read_parquet(f\"{DATA_DIR}/price_index.parquet\")\n",
|
||||
"estimates_df = pl.read_parquet(f\"{DATA_DIR}/estimated_prices.parquet\")\n",
|
||||
"backtest_df = pl.read_parquet(f\"{DATA_DIR}/backtest_results.parquet\")\n",
|
||||
"\n",
|
||||
"print(f\"Index: {len(index_df):,} rows, {index_df['sector'].n_unique():,} sectors\")\n",
|
||||
"print(f\"Estimates: {len(estimates_df):,} rows\")\n",
|
||||
"print(f\"Backtest: {len(backtest_df):,} rows\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Summary Metrics Table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def compute_metrics(actual, predicted):\n",
|
||||
" valid = np.isfinite(predicted) & np.isfinite(actual) & (actual > 0)\n",
|
||||
" a, p = actual[valid], predicted[valid]\n",
|
||||
" ape = np.abs(p - a) / a\n",
|
||||
" err = p - a\n",
|
||||
" return {\n",
|
||||
" \"MdAPE (%)\": f\"{np.median(ape)*100:.1f}\",\n",
|
||||
" \"% within 10%\": f\"{np.mean(ape <= 0.10)*100:.1f}\",\n",
|
||||
" \"% within 20%\": f\"{np.mean(ape <= 0.20)*100:.1f}\",\n",
|
||||
" \"% within 30%\": f\"{np.mean(ape <= 0.30)*100:.1f}\",\n",
|
||||
" \"MAE (\\u00a3)\": f\"{np.mean(np.abs(err)):,.0f}\",\n",
|
||||
" \"Mean signed error (\\u00a3)\": f\"{np.mean(err):+,.0f}\",\n",
|
||||
" \"n\": f\"{len(a):,}\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"actual = backtest_df[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"metrics = {\n",
|
||||
" \"Naive\": compute_metrics(actual, backtest_df[\"input_price\"].to_numpy().astype(np.float64)),\n",
|
||||
" \"Index\": compute_metrics(actual, backtest_df[\"predicted\"].to_numpy().astype(np.float64)),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"metrics_table = pl.DataFrame([\n",
|
||||
" {\"Metric\": k, **{stage: v[k] for stage, v in metrics.items()}}\n",
|
||||
" for k in list(metrics[\"Naive\"].keys())\n",
|
||||
"])\n",
|
||||
"metrics_table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. National + Sample Sector Index Curves"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# National index (average across all sectors weighted by n_pairs)\n",
|
||||
"national = (\n",
|
||||
" index_df\n",
|
||||
" .group_by(\"year\")\n",
|
||||
" .agg(\n",
|
||||
" (pl.col(\"log_index\") * pl.col(\"n_pairs\")).sum() / pl.col(\"n_pairs\").sum(),\n",
|
||||
" )\n",
|
||||
" .sort(\"year\")\n",
|
||||
" .with_columns(pl.lit(\"National\").alias(\"sector\"))\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Sample sectors: London, Manchester, rural, sparse\n",
|
||||
"sample_sectors = [\"EC1A 1\", \"SW1A 1\", \"M1 1\", \"LL55 4\"]\n",
|
||||
"available_sectors = index_df[\"sector\"].unique().to_list()\n",
|
||||
"sample_sectors = [s for s in sample_sectors if s in available_sectors]\n",
|
||||
"\n",
|
||||
"# If not enough, pick some with high/low n_pairs\n",
|
||||
"if len(sample_sectors) < 3:\n",
|
||||
" sector_counts = index_df.group_by(\"sector\").agg(pl.col(\"n_pairs\").first()).sort(\"n_pairs\", descending=True)\n",
|
||||
" top = sector_counts.head(2)[\"sector\"].to_list()\n",
|
||||
" bottom = sector_counts.filter(pl.col(\"n_pairs\") > 0).tail(2)[\"sector\"].to_list()\n",
|
||||
" sample_sectors = list(set(sample_sectors + top + bottom))[:5]\n",
|
||||
"\n",
|
||||
"samples = index_df.filter(pl.col(\"sector\").is_in(sample_sectors))\n",
|
||||
"\n",
|
||||
"combined = pl.concat([national.select(\"sector\", \"year\", \"log_index\"), samples.select(\"sector\", \"year\", \"log_index\")])\n",
|
||||
"\n",
|
||||
"# Normalize: index = 100 at base year (earliest available)\n",
|
||||
"combined = combined.with_columns(\n",
|
||||
" (pl.col(\"log_index\").exp() * 100).alias(\"index_100\"),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"fig = px.line(\n",
|
||||
" combined.to_pandas(), x=\"year\", y=\"index_100\", color=\"sector\",\n",
|
||||
" title=\"Repeat-Sales Price Index (base year = 100)\",\n",
|
||||
" labels={\"index_100\": \"Index (base=100)\", \"year\": \"Year\"},\n",
|
||||
")\n",
|
||||
"fig.update_layout(height=500)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. APE Distribution: Naive vs Index"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"actual = backtest_df[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"\n",
|
||||
"fig = go.Figure()\n",
|
||||
"for label, col in [(\"Naive\", \"input_price\"), (\"Index\", \"predicted\")]:\n",
|
||||
" pred = backtest_df[col].to_numpy().astype(np.float64)\n",
|
||||
" valid = np.isfinite(pred) & (actual > 0)\n",
|
||||
" ape = np.abs(pred[valid] - actual[valid]) / actual[valid]\n",
|
||||
" ape = ape[ape <= 1.0] # clip for display\n",
|
||||
" fig.add_trace(go.Histogram(x=ape * 100, name=label, opacity=0.6, nbinsx=100))\n",
|
||||
"\n",
|
||||
"fig.update_layout(\n",
|
||||
" title=\"Absolute Percentage Error Distribution\",\n",
|
||||
" xaxis_title=\"APE (%)\", yaxis_title=\"Count\",\n",
|
||||
" barmode=\"overlay\", height=500,\n",
|
||||
")\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Predicted vs Actual Scatter (log-log)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"np.random.seed(42)\n",
|
||||
"n_sample = min(10_000, len(backtest_df))\n",
|
||||
"idx = np.random.choice(len(backtest_df), n_sample, replace=False)\n",
|
||||
"sample = backtest_df[idx.tolist()]\n",
|
||||
"\n",
|
||||
"actual_sample = sample[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"pred = sample[\"predicted\"].to_numpy().astype(np.float64)\n",
|
||||
"\n",
|
||||
"fig = go.Figure()\n",
|
||||
"fig.add_trace(go.Scattergl(\n",
|
||||
" x=actual_sample, y=pred, mode=\"markers\",\n",
|
||||
" marker=dict(size=2, opacity=0.3), name=\"Index\",\n",
|
||||
"))\n",
|
||||
"# 45-degree reference line\n",
|
||||
"min_val = max(10_000, min(actual_sample.min(), np.nanmin(pred)))\n",
|
||||
"max_val = min(5_000_000, max(actual_sample.max(), np.nanmax(pred)))\n",
|
||||
"fig.add_trace(go.Scatter(\n",
|
||||
" x=[min_val, max_val], y=[min_val, max_val],\n",
|
||||
" mode=\"lines\", line=dict(color=\"red\", dash=\"dash\"), showlegend=False,\n",
|
||||
"))\n",
|
||||
"fig.update_xaxes(type=\"log\", title_text=\"Actual (\\u00a3)\")\n",
|
||||
"fig.update_yaxes(type=\"log\", title_text=\"Predicted (\\u00a3)\")\n",
|
||||
"fig.update_layout(title=\"Predicted vs Actual Price (log scale, 10K sample)\", height=500)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. MdAPE by Price Band"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"bands = [\n",
|
||||
" (0, 100_000, \"<100K\"),\n",
|
||||
" (100_000, 200_000, \"100-200K\"),\n",
|
||||
" (200_000, 300_000, \"200-300K\"),\n",
|
||||
" (300_000, 500_000, \"300-500K\"),\n",
|
||||
" (500_000, 1_000_000, \"500K-1M\"),\n",
|
||||
" (1_000_000, float(\"inf\"), \"1M+\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"actual = backtest_df[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"pred = backtest_df[\"predicted\"].to_numpy().astype(np.float64)\n",
|
||||
"naive = backtest_df[\"input_price\"].to_numpy().astype(np.float64)\n",
|
||||
"\n",
|
||||
"rows = []\n",
|
||||
"for lo, hi, label in bands:\n",
|
||||
" mask = (actual >= lo) & (actual < hi)\n",
|
||||
" if mask.sum() == 0:\n",
|
||||
" continue\n",
|
||||
" for name, arr in [(\"Naive\", naive), (\"Index\", pred)]:\n",
|
||||
" ape = np.abs(arr[mask] - actual[mask]) / actual[mask]\n",
|
||||
" valid = np.isfinite(ape)\n",
|
||||
" rows.append({\"Price Band\": label, \"Method\": name, \"MdAPE (%)\": float(np.median(ape[valid]) * 100)})\n",
|
||||
"\n",
|
||||
"band_df = pl.DataFrame(rows)\n",
|
||||
"fig = px.bar(\n",
|
||||
" band_df.to_pandas(), x=\"Price Band\", y=\"MdAPE (%)\", color=\"Method\",\n",
|
||||
" barmode=\"group\", title=\"MdAPE by Price Band\",\n",
|
||||
" category_orders={\"Price Band\": [b[2] for b in bands]},\n",
|
||||
")\n",
|
||||
"fig.update_layout(height=450)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. MdAPE by Region (Top 20 Postcode Areas)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"bt = backtest_df.with_columns(\n",
|
||||
" pl.col(\"sector\").str.replace(r\"\\d.*$\", \"\").alias(\"area\"),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Top 20 areas by volume\n",
|
||||
"top_areas = bt.group_by(\"area\").len().sort(\"len\", descending=True).head(20)[\"area\"].to_list()\n",
|
||||
"\n",
|
||||
"actual_np = bt[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"pred_np = bt[\"predicted\"].to_numpy().astype(np.float64)\n",
|
||||
"naive_np = bt[\"input_price\"].to_numpy().astype(np.float64)\n",
|
||||
"area_np = bt[\"area\"].to_numpy()\n",
|
||||
"\n",
|
||||
"rows = []\n",
|
||||
"for area in top_areas:\n",
|
||||
" mask = area_np == area\n",
|
||||
" a = actual_np[mask]\n",
|
||||
" for name, arr in [(\"Naive\", naive_np), (\"Index\", pred_np)]:\n",
|
||||
" p = arr[mask]\n",
|
||||
" valid = np.isfinite(p) & (a > 0)\n",
|
||||
" ape = np.abs(p[valid] - a[valid]) / a[valid]\n",
|
||||
" rows.append({\"Area\": area, \"Method\": name, \"MdAPE (%)\": float(np.median(ape) * 100)})\n",
|
||||
"\n",
|
||||
"area_df = pl.DataFrame(rows)\n",
|
||||
"fig = px.bar(\n",
|
||||
" area_df.to_pandas(), x=\"Area\", y=\"MdAPE (%)\", color=\"Method\",\n",
|
||||
" barmode=\"group\", title=\"MdAPE by Postcode Area (Top 20 by Volume)\",\n",
|
||||
" category_orders={\"Area\": top_areas},\n",
|
||||
")\n",
|
||||
"fig.update_layout(height=500)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. MdAPE by Holding Period"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"bt = backtest_df.with_columns(\n",
|
||||
" (pl.col(\"actual_year\") - pl.col(\"input_year\")).alias(\"gap_years\"),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"actual = bt[\"actual_price\"].to_numpy().astype(np.float64)\n",
|
||||
"pred = bt[\"predicted\"].to_numpy().astype(np.float64)\n",
|
||||
"naive = bt[\"input_price\"].to_numpy().astype(np.float64)\n",
|
||||
"gaps = bt[\"gap_years\"].to_numpy()\n",
|
||||
"max_gap = min(15, int(gaps.max()))\n",
|
||||
"\n",
|
||||
"rows = []\n",
|
||||
"for gap in range(1, max_gap + 1):\n",
|
||||
" mask = gaps == gap\n",
|
||||
" if mask.sum() < 100:\n",
|
||||
" continue\n",
|
||||
" a = actual[mask]\n",
|
||||
" for name, arr in [(\"Naive\", naive), (\"Index\", pred)]:\n",
|
||||
" p = arr[mask]\n",
|
||||
" valid = np.isfinite(p) & (a > 0)\n",
|
||||
" ape = np.abs(p[valid] - a[valid]) / a[valid]\n",
|
||||
" rows.append({\"Gap (years)\": gap, \"Method\": name, \"MdAPE (%)\": float(np.median(ape) * 100)})\n",
|
||||
"\n",
|
||||
"gap_df = pl.DataFrame(rows)\n",
|
||||
"fig = px.line(\n",
|
||||
" gap_df.to_pandas(), x=\"Gap (years)\", y=\"MdAPE (%)\", color=\"Method\",\n",
|
||||
" title=\"MdAPE by Holding Period (years between input and actual sale)\",\n",
|
||||
" markers=True,\n",
|
||||
")\n",
|
||||
"fig.update_layout(height=450)\n",
|
||||
"fig.show()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.12.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
167
pipeline/transform/price_backtest.py
Normal file
167
pipeline/transform/price_backtest.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Backtesting: Evaluate price index model on held-out recent sales.
|
||||
|
||||
Test set: properties with 2+ sales where the last sale is 2022-2025.
|
||||
Uses the second-to-last sale as input, predicts the last sale price.
|
||||
Compares index-based prediction against a naive baseline (raw input price).
|
||||
|
||||
Output: backtest_results.parquet with predictions vs actuals.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
CURRENT_YEAR = 2025
|
||||
TEST_YEAR_MIN = 2022
|
||||
|
||||
|
||||
def extract_test_set(input_path: Path) -> pl.DataFrame:
|
||||
"""Extract test pairs: second-to-last sale as input, last sale as ground truth."""
|
||||
print("Loading test set...")
|
||||
df = (
|
||||
pl.scan_parquet(input_path)
|
||||
.select("Postcode", "historical_prices")
|
||||
.filter(
|
||||
pl.col("Postcode").is_not_null(),
|
||||
pl.col("historical_prices").list.len() >= 2,
|
||||
)
|
||||
.with_columns(
|
||||
pl.col("Postcode").str.slice(0, pl.col("Postcode").str.len_chars() - 2).str.strip_chars().alias("sector"),
|
||||
# Last sale (ground truth)
|
||||
pl.col("historical_prices").list.last().struct.field("year").alias("actual_year"),
|
||||
pl.col("historical_prices").list.last().struct.field("price").alias("actual_price"),
|
||||
# Second-to-last sale (input)
|
||||
pl.col("historical_prices").list.get(-2).struct.field("year").alias("input_year"),
|
||||
pl.col("historical_prices").list.get(-2).struct.field("price").alias("input_price"),
|
||||
)
|
||||
.filter(
|
||||
pl.col("actual_year") >= TEST_YEAR_MIN,
|
||||
pl.col("input_price") > 0,
|
||||
pl.col("actual_price") > 0,
|
||||
pl.col("actual_year") > pl.col("input_year"),
|
||||
)
|
||||
.collect()
|
||||
)
|
||||
print(f" {len(df):,} test pairs (last sale {TEST_YEAR_MIN}-{CURRENT_YEAR})")
|
||||
return df
|
||||
|
||||
|
||||
def predict(test: pl.DataFrame, index: pl.DataFrame) -> pl.DataFrame:
|
||||
"""Index-based prediction: adjust input price by sector index change."""
|
||||
# Join index at input year
|
||||
test = test.join(
|
||||
index.select("sector", "year", pl.col("log_index").alias("log_index_input")),
|
||||
left_on=["sector", "input_year"],
|
||||
right_on=["sector", "year"],
|
||||
how="left",
|
||||
)
|
||||
# Join index at actual year
|
||||
test = test.join(
|
||||
index.select("sector", "year", pl.col("log_index").alias("log_index_actual")),
|
||||
left_on=["sector", "actual_year"],
|
||||
right_on=["sector", "year"],
|
||||
how="left",
|
||||
)
|
||||
|
||||
test = test.with_columns(
|
||||
(
|
||||
pl.col("input_price").cast(pl.Float64)
|
||||
* (pl.col("log_index_actual") - pl.col("log_index_input")).exp()
|
||||
).fill_null(pl.col("input_price").cast(pl.Float64)).alias("predicted"),
|
||||
)
|
||||
return test
|
||||
|
||||
|
||||
def compute_metrics(actual: np.ndarray, predicted: np.ndarray) -> dict:
|
||||
"""Compute error metrics."""
|
||||
valid = np.isfinite(predicted) & np.isfinite(actual) & (actual > 0)
|
||||
actual = actual[valid]
|
||||
predicted = predicted[valid]
|
||||
|
||||
ape = np.abs(predicted - actual) / actual
|
||||
signed_err = predicted - actual
|
||||
|
||||
return {
|
||||
"MdAPE (%)": float(np.median(ape) * 100),
|
||||
"% within 10%": float(np.mean(ape <= 0.10) * 100),
|
||||
"% within 20%": float(np.mean(ape <= 0.20) * 100),
|
||||
"% within 30%": float(np.mean(ape <= 0.30) * 100),
|
||||
"MAE (£)": float(np.mean(np.abs(signed_err))),
|
||||
"Mean signed error (£)": float(np.mean(signed_err)),
|
||||
"n": int(len(actual)),
|
||||
}
|
||||
|
||||
|
||||
def print_metrics_table(metrics_by_stage: dict):
|
||||
"""Print a comparison table of metrics."""
|
||||
print("\n" + "=" * 55)
|
||||
print("BACKTEST RESULTS")
|
||||
print("=" * 55)
|
||||
|
||||
metric_names = ["MdAPE (%)", "% within 10%", "% within 20%", "% within 30%", "MAE (£)", "Mean signed error (£)", "n"]
|
||||
stages = list(metrics_by_stage.keys())
|
||||
|
||||
# Header
|
||||
header = f"{'Metric':<25s}"
|
||||
for stage in stages:
|
||||
header += f" {stage:>14s}"
|
||||
print(header)
|
||||
print("-" * 55)
|
||||
|
||||
for metric in metric_names:
|
||||
row = f"{metric:<25s}"
|
||||
for stage in stages:
|
||||
val = metrics_by_stage[stage][metric]
|
||||
if metric == "n":
|
||||
row += f" {val:>14,d}"
|
||||
elif "£" in metric:
|
||||
row += f" {val:>13,.0f}"
|
||||
else:
|
||||
row += f" {val:>13.1f}%"
|
||||
print(row)
|
||||
|
||||
print("=" * 55)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backtest price estimation model")
|
||||
parser.add_argument("--input", type=Path, required=True, help="Path to wide.parquet")
|
||||
parser.add_argument("--index", type=Path, required=True, help="Path to price_index.parquet")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output backtest_results.parquet")
|
||||
args = parser.parse_args()
|
||||
|
||||
index = pl.read_parquet(args.index)
|
||||
print(f"Price index: {len(index):,} rows, {index['sector'].n_unique():,} sectors")
|
||||
|
||||
test = extract_test_set(args.input)
|
||||
|
||||
print("\nPredicting with price index...")
|
||||
test = predict(test, index)
|
||||
|
||||
# Compute and print metrics
|
||||
actual = test["actual_price"].to_numpy().astype(np.float64)
|
||||
metrics = {
|
||||
"Naive": compute_metrics(actual, test["input_price"].to_numpy().astype(np.float64)),
|
||||
"Index": compute_metrics(actual, test["predicted"].to_numpy().astype(np.float64)),
|
||||
}
|
||||
|
||||
print_metrics_table(metrics)
|
||||
|
||||
# Save results
|
||||
result = test.select(
|
||||
"Postcode", "sector",
|
||||
"input_year", "input_price",
|
||||
"actual_year", "actual_price",
|
||||
"predicted",
|
||||
)
|
||||
|
||||
result.write_parquet(args.output)
|
||||
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||
print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
|
||||
print(f" {len(result):,} rows")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
90
pipeline/transform/price_estimate.py
Normal file
90
pipeline/transform/price_estimate.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Apply repeat-sales price index to estimate current property prices.
|
||||
|
||||
Joins the precomputed price index (from price_index.py) with each property's
|
||||
last known sale to produce an inflation-adjusted current price estimate.
|
||||
|
||||
Output: estimated_prices.parquet with per-property estimates.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
CURRENT_YEAR = 2025
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Estimate current property prices")
|
||||
parser.add_argument("--input", type=Path, required=True, help="Path to wide.parquet")
|
||||
parser.add_argument("--index", type=Path, required=True, help="Path to price_index.parquet")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output estimated_prices.parquet")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Loading property data...")
|
||||
df = (
|
||||
pl.scan_parquet(args.input)
|
||||
.select("Postcode", "Address per Property Register", "Last known price", "Date of last transaction")
|
||||
.filter(
|
||||
pl.col("Last known price").is_not_null(),
|
||||
pl.col("Postcode").is_not_null(),
|
||||
)
|
||||
.with_columns(
|
||||
pl.col("Postcode").str.slice(0, pl.col("Postcode").str.len_chars() - 2).str.strip_chars().alias("sector"),
|
||||
pl.col("Date of last transaction").dt.year().alias("sale_year"),
|
||||
)
|
||||
.collect()
|
||||
)
|
||||
print(f" {len(df):,} properties with known price and postcode")
|
||||
|
||||
index = pl.read_parquet(args.index)
|
||||
print(f" Price index: {len(index):,} rows, {index['sector'].n_unique():,} sectors")
|
||||
|
||||
print("\nApplying repeat-sales index...")
|
||||
|
||||
# Join index at sale year
|
||||
df = df.join(
|
||||
index.select("sector", "year", pl.col("log_index").alias("log_index_sale")),
|
||||
left_on=["sector", "sale_year"],
|
||||
right_on=["sector", "year"],
|
||||
how="left",
|
||||
)
|
||||
|
||||
# Join index at current year
|
||||
index_current = (
|
||||
index.filter(pl.col("year") == CURRENT_YEAR)
|
||||
.select("sector", pl.col("log_index").alias("log_index_current"))
|
||||
)
|
||||
df = df.join(index_current, on="sector", how="left")
|
||||
|
||||
# Compute estimate; fall back to raw price when no index available
|
||||
df = df.with_columns(
|
||||
(
|
||||
pl.col("Last known price").cast(pl.Float64)
|
||||
* (pl.col("log_index_current") - pl.col("log_index_sale")).exp()
|
||||
)
|
||||
.fill_null(pl.col("Last known price").cast(pl.Float64))
|
||||
.alias("estimated_price"),
|
||||
)
|
||||
|
||||
n_adjusted = df.filter(pl.col("log_index_sale").is_not_null()).height
|
||||
print(f" {n_adjusted:,} properties adjusted by index ({n_adjusted / len(df) * 100:.1f}%)")
|
||||
|
||||
# Select output columns
|
||||
output = df.select(
|
||||
"Postcode",
|
||||
"Address per Property Register",
|
||||
pl.col("Last known price").alias("last_price"),
|
||||
"sale_year",
|
||||
"sector",
|
||||
"estimated_price",
|
||||
)
|
||||
|
||||
output.write_parquet(args.output)
|
||||
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||
print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
|
||||
print(f" {len(output):,} rows")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
272
pipeline/transform/price_index.py
Normal file
272
pipeline/transform/price_index.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
"""Stage 1: Repeat-Sales Price Index
|
||||
|
||||
Builds a hierarchical Case-Shiller repeat-sales price index from historical
|
||||
transaction data. Solves WLS regression per postcode sector, district, area,
|
||||
and nationally, then applies Bayesian shrinkage toward parent geographies.
|
||||
|
||||
Output: price_index.parquet with columns: sector, year, log_index, n_pairs
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from scipy.sparse import csc_matrix
|
||||
from scipy.sparse.linalg import lsqr
|
||||
from tqdm import tqdm
|
||||
|
||||
MIN_PAIRS = 5 # minimum pairs to compute an index
|
||||
SHRINKAGE_K = 50 # shrinkage parameter: higher = more shrinkage toward parent
|
||||
OUTLIER_THRESHOLD = 2.5 # |log_ratio| > this → drop (>12x price change)
|
||||
|
||||
|
||||
def extract_pairs(input_path: Path) -> pl.DataFrame:
|
||||
"""Extract consecutive sale pairs from historical_prices."""
|
||||
print("Loading historical prices...")
|
||||
df = (
|
||||
pl.scan_parquet(input_path)
|
||||
.select("Postcode", "historical_prices")
|
||||
.filter(
|
||||
pl.col("Postcode").is_not_null(),
|
||||
pl.col("historical_prices").list.len() >= 2,
|
||||
)
|
||||
.with_columns(
|
||||
pl.col("Postcode").str.slice(0, pl.col("Postcode").str.len_chars() - 2).str.strip_chars().alias("sector"),
|
||||
)
|
||||
.collect()
|
||||
)
|
||||
print(f" {len(df):,} properties with 2+ transactions")
|
||||
|
||||
print("Extracting consecutive pairs...")
|
||||
pairs = (
|
||||
df.lazy()
|
||||
.with_columns(
|
||||
pl.col("historical_prices").list.slice(0, pl.col("historical_prices").list.len() - 1).alias("from_txn"),
|
||||
pl.col("historical_prices").list.slice(1).alias("to_txn"),
|
||||
)
|
||||
.explode("from_txn", "to_txn")
|
||||
.with_columns(
|
||||
pl.col("from_txn").struct.field("year").alias("year1"),
|
||||
pl.col("from_txn").struct.field("price").alias("price1"),
|
||||
pl.col("to_txn").struct.field("year").alias("year2"),
|
||||
pl.col("to_txn").struct.field("price").alias("price2"),
|
||||
)
|
||||
.select("sector", "year1", "price1", "year2", "price2")
|
||||
.filter(
|
||||
pl.col("price1") > 0,
|
||||
pl.col("price2") > 0,
|
||||
pl.col("year2") > pl.col("year1"),
|
||||
)
|
||||
.with_columns(
|
||||
(pl.col("price2").cast(pl.Float64) / pl.col("price1").cast(pl.Float64)).log().alias("log_ratio"),
|
||||
(1.0 / (pl.col("year2") - pl.col("year1")).cast(pl.Float64).sqrt()).alias("weight"),
|
||||
)
|
||||
.filter(pl.col("log_ratio").abs() <= OUTLIER_THRESHOLD)
|
||||
.collect()
|
||||
)
|
||||
print(f" {len(pairs):,} consecutive pairs extracted")
|
||||
return pairs
|
||||
|
||||
|
||||
def solve_wls_index(years1: np.ndarray, years2: np.ndarray, log_ratios: np.ndarray, weights: np.ndarray) -> dict[int, float]:
|
||||
"""Solve WLS repeat-sales regression for a set of pairs.
|
||||
|
||||
Model: log(P2/P1) = beta[year2] - beta[year1], weighted by 1/sqrt(gap).
|
||||
Pin beta[min_year] = 0.
|
||||
Returns dict mapping year -> log_index (cumulative).
|
||||
"""
|
||||
if len(years1) < MIN_PAIRS:
|
||||
return {}
|
||||
|
||||
all_years = np.union1d(years1, years2)
|
||||
min_year = int(all_years.min())
|
||||
# Map years to column indices, skipping min_year (pinned to 0)
|
||||
col = 0
|
||||
year_to_col = {}
|
||||
for y in all_years:
|
||||
if int(y) != min_year:
|
||||
year_to_col[int(y)] = col
|
||||
col += 1
|
||||
n_cols = len(year_to_col)
|
||||
if n_cols == 0:
|
||||
return {}
|
||||
|
||||
n_rows = len(years1)
|
||||
row_idx = []
|
||||
col_idx = []
|
||||
data = []
|
||||
|
||||
for i in range(n_rows):
|
||||
y1, y2 = int(years1[i]), int(years2[i])
|
||||
if y2 in year_to_col:
|
||||
row_idx.append(i)
|
||||
col_idx.append(year_to_col[y2])
|
||||
data.append(weights[i])
|
||||
if y1 in year_to_col:
|
||||
row_idx.append(i)
|
||||
col_idx.append(year_to_col[y1])
|
||||
data.append(-weights[i])
|
||||
|
||||
A = csc_matrix((data, (row_idx, col_idx)), shape=(n_rows, n_cols))
|
||||
b = log_ratios * weights
|
||||
|
||||
result = lsqr(A, b, atol=1e-10, btol=1e-10)
|
||||
betas = result[0]
|
||||
|
||||
index = {min_year: 0.0}
|
||||
for year, col in year_to_col.items():
|
||||
index[year] = float(betas[col])
|
||||
return index
|
||||
|
||||
|
||||
def compute_indices_for_level(pairs: pl.DataFrame, group_col: str) -> dict[str, dict[int, float]]:
|
||||
"""Compute raw indices for each geographic group."""
|
||||
groups = pairs.group_by(group_col).agg(
|
||||
pl.col("year1"), pl.col("year2"), pl.col("log_ratio"), pl.col("weight"),
|
||||
)
|
||||
|
||||
indices = {}
|
||||
n_pairs_map = {}
|
||||
for row in tqdm(groups.iter_rows(named=True), total=len(groups), desc=f" Solving {group_col}"):
|
||||
key = row[group_col]
|
||||
y1 = np.array(row["year1"], dtype=np.int32)
|
||||
y2 = np.array(row["year2"], dtype=np.int32)
|
||||
lr = np.array(row["log_ratio"], dtype=np.float64)
|
||||
w = np.array(row["weight"], dtype=np.float64)
|
||||
idx = solve_wls_index(y1, y2, lr, w)
|
||||
if idx:
|
||||
indices[key] = idx
|
||||
n_pairs_map[key] = len(y1)
|
||||
return indices, n_pairs_map
|
||||
|
||||
|
||||
def shrink_index(raw: dict[int, float], parent: dict[int, float], n_pairs: int) -> dict[int, float]:
|
||||
"""Bayesian shrinkage toward parent index."""
|
||||
w = n_pairs / (n_pairs + SHRINKAGE_K)
|
||||
result = {}
|
||||
all_years = set(raw.keys()) | set(parent.keys())
|
||||
for y in all_years:
|
||||
raw_val = raw.get(y, parent.get(y, 0.0))
|
||||
parent_val = parent.get(y, raw.get(y, 0.0))
|
||||
result[y] = w * raw_val + (1 - w) * parent_val
|
||||
return result
|
||||
|
||||
|
||||
def forward_fill_index(index: dict[int, float], min_year: int, max_year: int) -> dict[int, float]:
|
||||
"""Forward-fill missing years so index is continuous."""
|
||||
filled = {}
|
||||
last_val = 0.0
|
||||
for y in range(min_year, max_year + 1):
|
||||
if y in index:
|
||||
last_val = index[y]
|
||||
filled[y] = last_val
|
||||
return filled
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build repeat-sales price index")
|
||||
parser.add_argument("--input", type=Path, required=True, help="Path to wide.parquet")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output price_index.parquet")
|
||||
args = parser.parse_args()
|
||||
|
||||
pairs = extract_pairs(args.input)
|
||||
|
||||
# Derive geographic hierarchy columns
|
||||
pairs = pairs.with_columns(
|
||||
# district = sector minus trailing digit(s), e.g. "SW1A 1" -> "SW1A"
|
||||
pl.col("sector").str.replace(r"\s+\d+$", "").alias("district"),
|
||||
).with_columns(
|
||||
# area = leading letters only, e.g. "SW1A" -> "SW"
|
||||
pl.col("district").str.replace(r"\d.*$", "").alias("area"),
|
||||
)
|
||||
|
||||
# Solve indices at each level
|
||||
print("\nComputing national index...")
|
||||
pairs_np = pairs.select("year1", "year2", "log_ratio", "weight")
|
||||
national_idx = solve_wls_index(
|
||||
pairs_np["year1"].to_numpy(),
|
||||
pairs_np["year2"].to_numpy(),
|
||||
pairs_np["log_ratio"].to_numpy(),
|
||||
pairs_np["weight"].to_numpy(),
|
||||
)
|
||||
print(f" National index: {len(national_idx)} years")
|
||||
|
||||
print("\nComputing area indices...")
|
||||
area_indices, area_pairs = compute_indices_for_level(pairs, "area")
|
||||
print(f" {len(area_indices)} areas with indices")
|
||||
|
||||
print("\nComputing district indices...")
|
||||
district_indices, district_pairs = compute_indices_for_level(pairs, "district")
|
||||
print(f" {len(district_indices)} districts with indices")
|
||||
|
||||
print("\nComputing sector indices...")
|
||||
sector_indices, sector_pairs = compute_indices_for_level(pairs, "sector")
|
||||
print(f" {len(sector_indices)} sectors with indices")
|
||||
|
||||
# Shrink area -> national
|
||||
print("\nApplying hierarchical shrinkage...")
|
||||
for area, idx in tqdm(area_indices.items(), desc=" Area shrinkage"):
|
||||
area_indices[area] = shrink_index(idx, national_idx, area_pairs[area])
|
||||
|
||||
# Shrink district -> area
|
||||
for dist, idx in tqdm(district_indices.items(), desc=" District shrinkage"):
|
||||
area = dist.replace(r"\d.*$", "")
|
||||
# Extract area from district (leading letters)
|
||||
area_key = ""
|
||||
for ch in dist:
|
||||
if ch.isalpha():
|
||||
area_key += ch
|
||||
else:
|
||||
break
|
||||
parent = area_indices.get(area_key, national_idx)
|
||||
district_indices[dist] = shrink_index(idx, parent, district_pairs[dist])
|
||||
|
||||
# Shrink sector -> district
|
||||
for sector, idx in tqdm(sector_indices.items(), desc=" Sector shrinkage"):
|
||||
# District = sector minus trailing space+digit
|
||||
dist_key = sector.rsplit(" ", 1)[0] if " " in sector else sector
|
||||
parent = district_indices.get(dist_key, national_idx)
|
||||
sector_indices[sector] = shrink_index(idx, parent, sector_pairs[sector])
|
||||
|
||||
# For sectors without enough data, fall back to district/area/national
|
||||
all_sectors = pairs["sector"].unique().to_list()
|
||||
min_year = int(pairs["year1"].min())
|
||||
max_year = max(int(pairs["year2"].max()), 2025)
|
||||
|
||||
print(f"\nFilling gaps and forward-filling ({min_year}-{max_year})...")
|
||||
rows = []
|
||||
for sector in tqdm(all_sectors, desc=" Forward-fill"):
|
||||
if sector in sector_indices:
|
||||
idx = sector_indices[sector]
|
||||
else:
|
||||
# Fall back to district, area, national
|
||||
dist_key = sector.rsplit(" ", 1)[0] if " " in sector else sector
|
||||
area_key = ""
|
||||
for ch in dist_key:
|
||||
if ch.isalpha():
|
||||
area_key += ch
|
||||
else:
|
||||
break
|
||||
idx = district_indices.get(dist_key, area_indices.get(area_key, national_idx))
|
||||
|
||||
n = sector_pairs.get(sector, 0)
|
||||
filled = forward_fill_index(idx, min_year, max_year)
|
||||
for year, log_idx in filled.items():
|
||||
rows.append((sector, year, log_idx, n))
|
||||
|
||||
result = pl.DataFrame(
|
||||
rows,
|
||||
schema={"sector": pl.String, "year": pl.Int32, "log_index": pl.Float64, "n_pairs": pl.Int64},
|
||||
orient="row",
|
||||
)
|
||||
|
||||
result = result.sort("sector", "year")
|
||||
result.write_parquet(args.output)
|
||||
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||
print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
|
||||
print(f" {result['sector'].n_unique():,} sectors × {max_year - min_year + 1} years = {len(result):,} rows")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -3,7 +3,7 @@ import { ScreenshotCache } from './cache.js';
|
|||
import { takeScreenshot, checkWebGL, closeBrowser } from './screenshot.js';
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '8002', 10);
|
||||
const NARROWIT_URL = process.env.NARROWIT_URL || 'http://localhost:8001';
|
||||
const APP_URL = process.env.APP_URL || 'http://localhost:8001';
|
||||
const CACHE_DIR = process.env.CACHE_DIR || '/cache';
|
||||
|
||||
const cache = new ScreenshotCache(CACHE_DIR);
|
||||
|
|
@ -47,7 +47,7 @@ app.get('/screenshot', async (req, res) => {
|
|||
// Build the URL for the frontend in screenshot mode
|
||||
const qs = new URLSearchParams(params);
|
||||
qs.set('screenshot', '1');
|
||||
const url = `${NARROWIT_URL}/?${qs}`;
|
||||
const url = `${APP_URL}/?${qs}`;
|
||||
|
||||
console.log(`Taking screenshot: ${url}`);
|
||||
const png = await takeScreenshot(url);
|
||||
|
|
@ -67,7 +67,7 @@ app.get('/screenshot', async (req, res) => {
|
|||
|
||||
const server = app.listen(PORT, () => {
|
||||
console.log(`Screenshot service listening on port ${PORT}`);
|
||||
console.log(` NARROWIT_URL: ${NARROWIT_URL}`);
|
||||
console.log(` APP_URL: ${APP_URL}`);
|
||||
console.log(` CACHE_DIR: ${CACHE_DIR}`);
|
||||
});
|
||||
|
||||
|
|
|
|||
70
server-rs/src/aggregation.rs
Normal file
70
server-rs/src/aggregation.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
/// Per-cell accumulator for aggregating features (min/max/sum/count).
|
||||
/// Uses Box<[T]> instead of Vec<T> to avoid storing capacity (saves 8 bytes per field per cell).
|
||||
/// Shared by hexagon and postcode aggregation routes.
|
||||
pub struct Aggregator {
|
||||
pub count: u32,
|
||||
pub mins: Box<[f32]>,
|
||||
pub maxs: Box<[f32]>,
|
||||
pub sums: Box<[f64]>,
|
||||
pub feat_counts: Box<[u32]>,
|
||||
}
|
||||
|
||||
impl Aggregator {
|
||||
pub fn new(num_features: usize) -> Self {
|
||||
Aggregator {
|
||||
count: 0,
|
||||
mins: vec![f32::INFINITY; num_features].into_boxed_slice(),
|
||||
maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(),
|
||||
sums: vec![0.0f64; num_features].into_boxed_slice(),
|
||||
feat_counts: vec![0u32; num_features].into_boxed_slice(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a row using row-major feature_data layout.
|
||||
/// feature_data[row * num_features + feat_idx] — all features for one row
|
||||
/// are contiguous, so this reads a single cache line per ~8 features.
|
||||
#[inline]
|
||||
pub fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
for (feat_index, &value) in row_slice.iter().enumerate() {
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a row, only aggregating the features at the given indices.
|
||||
#[inline]
|
||||
pub fn add_row_selective(
|
||||
&mut self,
|
||||
feature_data: &[f32],
|
||||
row: usize,
|
||||
num_features: usize,
|
||||
indices: &[usize],
|
||||
) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
for &feat_index in indices {
|
||||
let value = feature_data[base + feat_index];
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,6 +11,8 @@ pub const MAX_POIS_PER_REQUEST: usize = 2500;
|
|||
pub const MAX_CELLS_PER_REQUEST: usize = 5000;
|
||||
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
|
||||
pub const MAX_PROPERTIES_LIMIT: usize = 500;
|
||||
pub const MAX_PRICE_HISTORY_POINTS: usize = 5000;
|
||||
pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02;
|
||||
|
||||
pub const AREA_SUMMARY_SYSTEM_PROMPT: &str = "You are an experienced estate agent with an expertise in area analysis. Help the user find his/her dream area or perfect postcode to settle in. The user is looking to buy a property based on the filters they provide. Given area statistics, write at most a single concise sentences summarising the key characteristics of the area. Be factual and highlight notable values. Do not use bullet points or headers — just flowing prose. Do not use markdown formatting. Highlight unusual facts that stand out from the average, but do not exaggerate. If there are no notable characteristics, say so. Always write at most a single sentence! Reason about the relation of different statistics to each other.";
|
||||
pub const AREA_SUMMARY_MAX_TOKENS: usize = 300;
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@ mod property;
|
|||
|
||||
pub use poi::{POICategoryGroup, POIData};
|
||||
pub use postcodes::PostcodeData;
|
||||
pub use property::{precompute_h3, Histogram, PropertyData};
|
||||
pub use property::{precompute_h3, FeatureStats, Histogram, PropertyData};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
mod aggregation;
|
||||
mod auth;
|
||||
mod consts;
|
||||
mod data;
|
||||
|
|
@ -28,7 +29,7 @@ use tracing_subscriber::EnvFilter;
|
|||
use state::AppState;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "narrowit", about = "Narrowit property map server")]
|
||||
#[command(name = "perfect-postcodes", about = "Perfect Postcodes property map server")]
|
||||
struct Cli {
|
||||
/// Path to the wide property parquet file
|
||||
#[arg(long)]
|
||||
|
|
@ -58,7 +59,7 @@ struct Cli {
|
|||
#[arg(
|
||||
long,
|
||||
env = "PUBLIC_URL",
|
||||
default_value = "https://narrowit.schmelczer.dev"
|
||||
default_value = "https://perfectpostcodes.schmelczer.dev"
|
||||
)]
|
||||
public_url: String,
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use axum::response::Response;
|
|||
|
||||
use crate::state::AppState;
|
||||
|
||||
const OG_PLACEHOLDER: &str = r#"<meta name="x-og-placeholder" content="__NARROWIT_OG_TAGS__"/>"#;
|
||||
const OG_PLACEHOLDER: &str = r#"<meta name="x-og-placeholder" content="__PERFECT_POSTCODES_OG_TAGS__"/>"#;
|
||||
|
||||
pub async fn og_middleware(request: Request, next: Next) -> Response {
|
||||
// Capture the query string before passing the request through
|
||||
|
|
@ -47,14 +47,14 @@ pub async fn og_middleware(request: Request, next: Next) -> Response {
|
|||
};
|
||||
|
||||
let og_tags = format!(
|
||||
r#"<meta property="og:title" content="Narrowit — Every neighbourhood in England & Wales" />
|
||||
r#"<meta property="og:title" content="Perfect Postcodes — Every neighbourhood in England & Wales" />
|
||||
<meta property="og:description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />
|
||||
<meta property="og:type" content="website" />
|
||||
<meta property="og:image" content="{og_image_url}" />
|
||||
<meta property="og:image:width" content="1200" />
|
||||
<meta property="og:image:height" content="630" />
|
||||
<meta name="twitter:card" content="summary_large_image" />
|
||||
<meta name="twitter:title" content="Narrowit — Every neighbourhood in England & Wales" />
|
||||
<meta name="twitter:title" content="Perfect Postcodes — Every neighbourhood in England & Wales" />
|
||||
<meta name="twitter:description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />"#
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
mod bounds;
|
||||
mod fields;
|
||||
mod filters;
|
||||
mod h3;
|
||||
|
||||
pub use bounds::{bounds_intersect, h3_cell_bounds, parse_bounds};
|
||||
pub use bounds::{bounds_intersect, h3_cell_bounds, parse_bounds, require_bounds};
|
||||
pub use fields::{parse_field_indices, parse_field_set};
|
||||
pub use filters::{parse_filters, row_passes_filters, ParsedEnumFilter, ParsedFilter};
|
||||
pub use h3::{cell_for_row, needs_parent, validate_h3_resolution};
|
||||
|
|
|
|||
|
|
@ -48,6 +48,17 @@ pub fn h3_cell_bounds(cell: h3o::CellIndex, buffer: f64) -> (f64, f64, f64, f64)
|
|||
)
|
||||
}
|
||||
|
||||
/// Require a bounds parameter, returning an error if absent, then parse it.
|
||||
pub fn require_bounds(
|
||||
bounds: Option<String>,
|
||||
) -> Result<(f64, f64, f64, f64), (StatusCode, String)> {
|
||||
let bounds_str = bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
parse_bounds(&bounds_str)
|
||||
}
|
||||
|
||||
pub fn parse_bounds(bounds_str: &str) -> Result<(f64, f64, f64, f64), (StatusCode, String)> {
|
||||
let parts: Vec<f64> = bounds_str
|
||||
.split(',')
|
||||
|
|
|
|||
42
server-rs/src/parsing/fields.rs
Normal file
42
server-rs/src/parsing/fields.rs
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
use std::collections::HashSet;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
/// Parse an optional `?fields=` query param into feature indices for selective aggregation.
|
||||
/// Returns `None` if fields is `None` (all features included), or `Some(indices)` if specified.
|
||||
pub fn parse_field_indices(
|
||||
fields: Option<&str>,
|
||||
name_to_index: &FxHashMap<String, usize>,
|
||||
) -> Option<Vec<usize>> {
|
||||
fields.map(|fields_str| {
|
||||
if fields_str.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
fields_str
|
||||
.split(',')
|
||||
.filter_map(|name| {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
name_to_index.get(name).copied()
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse an optional `?fields=` query param into a HashSet for stats filtering.
|
||||
/// Returns `(fields_specified, field_set)`.
|
||||
pub fn parse_field_set(fields: Option<&str>) -> (bool, HashSet<String>) {
|
||||
let fields_specified = fields.is_some();
|
||||
let field_set: HashSet<String> = fields
|
||||
.map(|fields_str| {
|
||||
fields_str
|
||||
.split(',')
|
||||
.map(|field| field.trim().to_string())
|
||||
.filter(|field| !field.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
(fields_specified, field_set)
|
||||
}
|
||||
52
server-rs/src/parsing/h3.rs
Normal file
52
server-rs/src/parsing/h3.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
use axum::http::StatusCode;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::consts::{H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN};
|
||||
|
||||
/// Validate that an H3 resolution is within the allowed range and convert to h3o::Resolution.
|
||||
pub fn validate_h3_resolution(resolution: u8) -> Result<h3o::Resolution, (StatusCode, String)> {
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
h3o::Resolution::try_from(resolution).map_err(|error| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("Invalid H3 resolution {}: {}", resolution, error),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve a row's H3 cell at the requested resolution, using precomputed max-resolution cells.
|
||||
#[inline]
|
||||
pub fn cell_for_row(
|
||||
row: usize,
|
||||
precomputed: &[u64],
|
||||
h3_res: h3o::Resolution,
|
||||
need_parent: bool,
|
||||
) -> u64 {
|
||||
let max_cell = precomputed[row];
|
||||
if !need_parent || max_cell == 0 {
|
||||
return max_cell;
|
||||
}
|
||||
h3o::CellIndex::try_from(max_cell)
|
||||
.ok()
|
||||
.and_then(|ci| ci.parent(h3_res))
|
||||
.map(u64::from)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Whether the given resolution requires computing a parent from precomputed cells.
|
||||
#[inline]
|
||||
pub fn needs_parent(resolution: u8) -> bool {
|
||||
resolution < H3_PRECOMPUTE_MAX
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ mod postcode_stats;
|
|||
mod postcodes;
|
||||
pub(crate) mod properties;
|
||||
mod screenshot;
|
||||
mod stats;
|
||||
mod tiles;
|
||||
|
||||
pub use area_summary::post_area_summary;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
|
|||
use serde::Deserialize;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::parsing::{parse_bounds, parse_filters, row_passes_filters};
|
||||
use crate::parsing::{parse_field_indices, parse_filters, require_bounds, row_passes_filters};
|
||||
use crate::routes::FeatureInfo;
|
||||
use crate::state::AppState;
|
||||
|
||||
|
|
@ -135,12 +135,7 @@ pub async fn get_export(
|
|||
state: Arc<AppState>,
|
||||
Query(params): Query<ExportParams>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let bounds_str = params.bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
|
||||
let (south, west, north, east) = parse_bounds(&bounds_str)?;
|
||||
let (south, west, north, east) = require_bounds(params.bounds)?;
|
||||
|
||||
let filters_str = params.filters.clone();
|
||||
let fields_str = params.fields.clone();
|
||||
|
|
@ -234,7 +229,10 @@ pub async fn get_export(
|
|||
let was_sampled = postcode_aggs.len() > MAX_EXPORT_POSTCODES;
|
||||
if was_sampled {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
bounds_str.hash(&mut hasher);
|
||||
south.to_bits().hash(&mut hasher);
|
||||
west.to_bits().hash(&mut hasher);
|
||||
north.to_bits().hash(&mut hasher);
|
||||
east.to_bits().hash(&mut hasher);
|
||||
let seed = hasher.finish();
|
||||
|
||||
let len = postcode_aggs.len();
|
||||
|
|
@ -251,20 +249,8 @@ pub async fn get_export(
|
|||
// Determine column order: filter features first, then remaining
|
||||
let filter_feature_names = extract_filter_feature_names(filters_str.as_deref());
|
||||
|
||||
let field_indices: Option<Vec<usize>> = fields_str.as_ref().map(|fs| {
|
||||
if fs.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
fs.split(',')
|
||||
.filter_map(|name| {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
state.feature_name_to_index.get(name).copied()
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
let field_indices =
|
||||
parse_field_indices(fields_str.as_deref(), &state.feature_name_to_index);
|
||||
|
||||
let all_feature_indices: Vec<usize> = if let Some(ref indices) = field_indices {
|
||||
indices.clone()
|
||||
|
|
@ -314,7 +300,7 @@ pub async fn get_export(
|
|||
.set_font_color("#666666")
|
||||
.set_align(FormatAlign::Left);
|
||||
|
||||
// Row 0: "View on Narrowit" link
|
||||
// Row 0: "View on Perfect Postcodes" link
|
||||
let mut dashboard_url = format!("{}/", public_url);
|
||||
let mut query_parts: Vec<String> = Vec::new();
|
||||
query_parts.push(format!("v={}", view_param));
|
||||
|
|
@ -329,7 +315,7 @@ pub async fn get_export(
|
|||
}
|
||||
|
||||
sheet
|
||||
.write_url(0, 0, Url::new(&dashboard_url).set_text("View on Narrowit"))
|
||||
.write_url(0, 0, Url::new(&dashboard_url).set_text("View on Perfect Postcodes"))
|
||||
.map_err(|err| format!("Failed to write URL: {err}"))?;
|
||||
sheet
|
||||
.set_row_format(0, &link_fmt)
|
||||
|
|
@ -499,7 +485,7 @@ pub async fn get_export(
|
|||
),
|
||||
(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"narrowit-export.xlsx\"",
|
||||
"attachment; filename=\"perfect-postcodes-export.xlsx\"",
|
||||
),
|
||||
],
|
||||
bytes,
|
||||
|
|
|
|||
|
|
@ -8,10 +8,14 @@ use axum::response::Json;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN};
|
||||
use crate::parsing::{h3_cell_bounds, parse_filters, row_passes_filters};
|
||||
use crate::parsing::{
|
||||
cell_for_row, h3_cell_bounds, needs_parent, parse_field_set, parse_filters, row_passes_filters,
|
||||
validate_h3_resolution,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
use super::stats;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HistogramStats {
|
||||
pub min: f64,
|
||||
|
|
@ -78,19 +82,8 @@ pub async fn get_hexagon_stats(
|
|||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
validate_h3_resolution(resolution)?;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
|
|
@ -100,48 +93,25 @@ pub async fn get_hexagon_stats(
|
|||
);
|
||||
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
|
||||
|
||||
let fields_specified = params.fields.is_some();
|
||||
let field_set: std::collections::HashSet<String> = params
|
||||
.fields
|
||||
.as_ref()
|
||||
.map(|fields_str| {
|
||||
fields_str
|
||||
.split(',')
|
||||
.map(|field| field.trim().to_string())
|
||||
.filter(|field| !field.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let (fields_specified, field_set) = parse_field_set(params.fields.as_deref());
|
||||
|
||||
let response = tokio::task::spawn_blocking(move || {
|
||||
let start_time = std::time::Instant::now();
|
||||
let precomputed = &state.h3_cells;
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let need_parent = resolution < H3_PRECOMPUTE_MAX;
|
||||
let need_parent = needs_parent(resolution);
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
|
||||
let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(cell, 0.001);
|
||||
|
||||
let cell_for_row = |row: usize| -> u64 {
|
||||
let max_cell = precomputed[row];
|
||||
if !need_parent || max_cell == 0 {
|
||||
return max_cell;
|
||||
}
|
||||
h3o::CellIndex::try_from(max_cell)
|
||||
.ok()
|
||||
.and_then(|ci| ci.parent(h3_res))
|
||||
.map(u64::from)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let mut matching_rows: Vec<usize> = Vec::new();
|
||||
state
|
||||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if cell_for_row(row) == cell_u64
|
||||
if cell_for_row(row, precomputed, h3_res, need_parent) == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -156,149 +126,23 @@ pub async fn get_hexagon_stats(
|
|||
|
||||
let total_count = matching_rows.len();
|
||||
|
||||
// Collect price history (year, price) pairs
|
||||
let price_history = {
|
||||
let year_idx = state
|
||||
.feature_name_to_index
|
||||
.get("Date of last transaction")
|
||||
.copied();
|
||||
let price_idx = state.feature_name_to_index.get("Last known price").copied();
|
||||
match (year_idx, price_idx) {
|
||||
(Some(yi), Some(pi)) => {
|
||||
let mut points: Vec<PricePoint> = matching_rows
|
||||
.iter()
|
||||
.filter_map(|&row| {
|
||||
let year = feature_data[row * num_features + yi];
|
||||
let price = feature_data[row * num_features + pi];
|
||||
if year.is_finite() && price.is_finite() {
|
||||
Some(PricePoint { year, price })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// Cap at 5000 points by evenly sampling
|
||||
if points.len() > 5000 {
|
||||
let step = points.len() as f64 / 5000.0;
|
||||
points = (0..5000)
|
||||
.map(|i| {
|
||||
let idx = (i as f64 * step) as usize;
|
||||
PricePoint {
|
||||
year: points[idx].year,
|
||||
price: points[idx].price,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
points
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
};
|
||||
let price_history = stats::extract_price_history(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
num_features,
|
||||
&state.feature_name_to_index,
|
||||
);
|
||||
|
||||
let mut numeric_features = Vec::new();
|
||||
let mut enum_features_out = Vec::new();
|
||||
|
||||
for (feature_index, feature_name) in state.data.feature_names.iter().enumerate() {
|
||||
if fields_specified && !field_set.contains(feature_name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is an enum feature
|
||||
if let Some(enum_values) = state.data.enum_values.get(&feature_index) {
|
||||
// Enum feature: count occurrences of each value
|
||||
let mut value_counts = vec![0u64; enum_values.len()];
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
let idx = value as usize;
|
||||
if idx < value_counts.len() {
|
||||
value_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let counts: HashMap<String, u64> = value_counts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, &count)| count > 0)
|
||||
.map(|(idx, &count)| (enum_values[idx].clone(), count))
|
||||
.collect();
|
||||
|
||||
if !counts.is_empty() {
|
||||
enum_features_out.push(EnumFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
counts,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Numeric feature: compute stats and histogram
|
||||
let global_hist = &state.data.feature_stats[feature_index].histogram;
|
||||
let p1 = global_hist.p1;
|
||||
let p99 = global_hist.p99;
|
||||
// Use same bin count as global histogram for consistency
|
||||
let num_bins = global_hist.counts.len();
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut bins = vec![0u64; num_bins];
|
||||
|
||||
// Compute middle bin width (between p1 and p99)
|
||||
let middle_bins = num_bins.saturating_sub(2);
|
||||
let middle_width = if middle_bins > 0 && p99 > p1 {
|
||||
(p99 - p1) / middle_bins as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
count += 1;
|
||||
if value < min_value {
|
||||
min_value = value;
|
||||
}
|
||||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value as f64;
|
||||
|
||||
// Bin using p1/p99 outlier structure
|
||||
let bin = if value < p1 {
|
||||
0 // Low outlier bin
|
||||
} else if value >= p99 {
|
||||
num_bins - 1 // High outlier bin
|
||||
} else if middle_width > 0.0 {
|
||||
// Middle bins (1 to n-2)
|
||||
let middle_bin = ((value - p1) / middle_width) as usize;
|
||||
(1 + middle_bin).min(num_bins - 2)
|
||||
} else {
|
||||
num_bins / 2 // Fallback if p1 == p99
|
||||
};
|
||||
bins[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
numeric_features.push(NumericFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
count,
|
||||
min: min_value as f64,
|
||||
max: max_value as f64,
|
||||
mean: sum / count as f64,
|
||||
histogram: HistogramStats {
|
||||
min: global_hist.min as f64,
|
||||
max: global_hist.max as f64,
|
||||
p1: p1 as f64,
|
||||
p99: p99 as f64,
|
||||
counts: bins,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
&state.data.feature_names,
|
||||
num_features,
|
||||
&state.data.enum_values,
|
||||
&state.data.feature_stats,
|
||||
fields_specified,
|
||||
&field_set,
|
||||
);
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(
|
||||
|
|
|
|||
|
|
@ -6,11 +6,13 @@ use axum::response::Json;
|
|||
use rustc_hash::FxHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use tracing::{info, warn};
|
||||
use tracing::info;
|
||||
|
||||
use crate::consts::{H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_CELLS_PER_REQUEST};
|
||||
use crate::aggregation::Aggregator;
|
||||
use crate::consts::MAX_CELLS_PER_REQUEST;
|
||||
use crate::parsing::{
|
||||
bounds_intersect, h3_cell_bounds, parse_bounds, parse_filters, row_passes_filters,
|
||||
bounds_intersect, cell_for_row, h3_cell_bounds, needs_parent, parse_field_indices,
|
||||
parse_filters, require_bounds, row_passes_filters, validate_h3_resolution,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
|
|
@ -32,79 +34,9 @@ pub struct HexagonParams {
|
|||
fields: Option<String>,
|
||||
}
|
||||
|
||||
/// Per-cell accumulator for aggregating features.
|
||||
/// Uses Box<[T]> instead of Vec<T> to avoid storing capacity (saves 8 bytes per field per cell).
|
||||
struct CellAgg {
|
||||
count: u32,
|
||||
mins: Box<[f32]>,
|
||||
maxs: Box<[f32]>,
|
||||
sums: Box<[f64]>,
|
||||
feat_counts: Box<[u32]>,
|
||||
}
|
||||
|
||||
impl CellAgg {
|
||||
fn new(num_features: usize) -> Self {
|
||||
CellAgg {
|
||||
count: 0,
|
||||
mins: vec![f32::INFINITY; num_features].into_boxed_slice(),
|
||||
maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(),
|
||||
sums: vec![0.0f64; num_features].into_boxed_slice(),
|
||||
feat_counts: vec![0u32; num_features].into_boxed_slice(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a row using row-major feature_data layout.
|
||||
/// feature_data[row * num_features + feat_idx] — all features for one row
|
||||
/// are contiguous, so this reads a single cache line per ~8 features.
|
||||
#[inline]
|
||||
fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
for (feat_index, &value) in row_slice.iter().enumerate() {
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a row, only aggregating the features at the given indices.
|
||||
#[inline]
|
||||
fn add_row_selective(
|
||||
&mut self,
|
||||
feature_data: &[f32],
|
||||
row: usize,
|
||||
num_features: usize,
|
||||
indices: &[usize],
|
||||
) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
for &feat_index in indices {
|
||||
let value = feature_data[base + feat_index];
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build feature maps from aggregated cell data, filtering to only cells that intersect the query bounds.
|
||||
fn build_feature_maps(
|
||||
groups: &FxHashMap<u64, CellAgg>,
|
||||
groups: &FxHashMap<u64, Aggregator>,
|
||||
min_keys: &[String],
|
||||
max_keys: &[String],
|
||||
avg_keys: &[String],
|
||||
|
|
@ -172,26 +104,9 @@ pub async fn get_hexagons(
|
|||
Query(params): Query<HexagonParams>,
|
||||
) -> Result<Json<HexagonsResponse>, (StatusCode, String)> {
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
validate_h3_resolution(resolution)?;
|
||||
|
||||
let bounds_str = params.bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
|
||||
let (south, west, north, east) = parse_bounds(&bounds_str)?;
|
||||
let (south, west, north, east) = require_bounds(params.bounds)?;
|
||||
|
||||
let filters_str = params.filters.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
|
|
@ -201,24 +116,7 @@ pub async fn get_hexagons(
|
|||
);
|
||||
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
|
||||
|
||||
// Parse optional `fields` param into feature indices.
|
||||
// If `fields` is absent (None), all features are included.
|
||||
// If `fields` is present (even empty string), only listed features are included.
|
||||
let field_indices: Option<Vec<usize>> = params.fields.as_ref().map(|fields_str| {
|
||||
if fields_str.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
fields_str
|
||||
.split(',')
|
||||
.filter_map(|name| {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
state.feature_name_to_index.get(name).copied()
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index);
|
||||
|
||||
let response = tokio::task::spawn_blocking(move || -> Result<HexagonsResponse, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
|
|
@ -232,21 +130,9 @@ pub async fn get_hexagons(
|
|||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|error| format!("Invalid H3 resolution {}: {}", resolution, error))?;
|
||||
let precomputed = &state.h3_cells;
|
||||
let need_parent = resolution < H3_PRECOMPUTE_MAX;
|
||||
let need_parent = needs_parent(resolution);
|
||||
|
||||
let mut groups: FxHashMap<u64, CellAgg> = FxHashMap::default();
|
||||
|
||||
let cell_for_row = |row: usize| -> u64 {
|
||||
let max_cell = precomputed[row];
|
||||
if !need_parent || max_cell == 0 {
|
||||
return max_cell;
|
||||
}
|
||||
h3o::CellIndex::try_from(max_cell)
|
||||
.ok()
|
||||
.and_then(|ci| ci.parent(h3_res))
|
||||
.map(u64::from)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
let mut groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
|
||||
|
||||
// Hoist has_selective branch outside the hot loop to avoid per-row branching
|
||||
if let Some(sel_indices) = field_indices.as_deref() {
|
||||
|
|
@ -263,10 +149,10 @@ pub async fn get_hexagons(
|
|||
) {
|
||||
return;
|
||||
}
|
||||
let cell_id = cell_for_row(row);
|
||||
let cell_id = cell_for_row(row, precomputed, h3_res, need_parent);
|
||||
let aggregation = groups
|
||||
.entry(cell_id)
|
||||
.or_insert_with(|| CellAgg::new(num_features));
|
||||
.or_insert_with(|| Aggregator::new(num_features));
|
||||
aggregation.add_row_selective(feature_data, row, num_features, sel_indices);
|
||||
});
|
||||
} else {
|
||||
|
|
@ -283,10 +169,10 @@ pub async fn get_hexagons(
|
|||
) {
|
||||
return;
|
||||
}
|
||||
let cell_id = cell_for_row(row);
|
||||
let cell_id = cell_for_row(row, precomputed, h3_res, need_parent);
|
||||
let aggregation = groups
|
||||
.entry(cell_id)
|
||||
.or_insert_with(|| CellAgg::new(num_features));
|
||||
.or_insert_with(|| Aggregator::new(num_features));
|
||||
aggregation.add_row(feature_data, row, num_features);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use tracing::info;
|
|||
|
||||
use crate::consts::MAX_POIS_PER_REQUEST;
|
||||
use crate::data::POICategoryGroup;
|
||||
use crate::parsing::parse_bounds;
|
||||
use crate::parsing::require_bounds;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -39,12 +39,7 @@ pub async fn get_pois(
|
|||
state: Arc<AppState>,
|
||||
Query(params): Query<POIParams>,
|
||||
) -> Result<Json<POIsResponse>, (StatusCode, String)> {
|
||||
let bounds_str = params.bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
|
||||
let (south, west, north, east) = parse_bounds(&bounds_str)?;
|
||||
let (south, west, north, east) = require_bounds(params.bounds)?;
|
||||
|
||||
let categories_str = params.categories.clone();
|
||||
let category_filter: Option<rustc_hash::FxHashSet<String>> = params
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::Query;
|
||||
|
|
@ -7,12 +6,12 @@ use axum::response::Json;
|
|||
use serde::Deserialize;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::parsing::{parse_filters, row_passes_filters};
|
||||
use crate::consts::POSTCODE_SEARCH_OFFSET;
|
||||
use crate::parsing::{parse_field_set, parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
||||
use super::hexagon_stats::{
|
||||
EnumFeatureStats, HexagonStatsResponse, HistogramStats, NumericFeatureStats, PricePoint,
|
||||
};
|
||||
use super::hexagon_stats::HexagonStatsResponse;
|
||||
use super::stats;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PostcodeStatsParams {
|
||||
|
|
@ -56,18 +55,7 @@ pub async fn get_postcode_stats(
|
|||
);
|
||||
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
|
||||
|
||||
let fields_specified = params.fields.is_some();
|
||||
let field_set: std::collections::HashSet<String> = params
|
||||
.fields
|
||||
.as_ref()
|
||||
.map(|fields_str| {
|
||||
fields_str
|
||||
.split(',')
|
||||
.map(|field| field.trim().to_string())
|
||||
.filter(|field| !field.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let (fields_specified, field_set) = parse_field_set(params.fields.as_deref());
|
||||
|
||||
let postcode_str = normalized.clone();
|
||||
|
||||
|
|
@ -76,8 +64,8 @@ pub async fn get_postcode_stats(
|
|||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
|
||||
// Search ±0.02° around centroid (~2km, generous for a postcode)
|
||||
let offset: f64 = 0.02;
|
||||
// Search around centroid (generous for a postcode)
|
||||
let offset: f64 = POSTCODE_SEARCH_OFFSET;
|
||||
let min_lat = centroid_lat as f64 - offset;
|
||||
let max_lat = centroid_lat as f64 + offset;
|
||||
let min_lon = centroid_lon as f64 - offset;
|
||||
|
|
@ -104,144 +92,23 @@ pub async fn get_postcode_stats(
|
|||
|
||||
let total_count = matching_rows.len();
|
||||
|
||||
// Collect price history (year, price) pairs
|
||||
let price_history = {
|
||||
let year_idx = state
|
||||
.feature_name_to_index
|
||||
.get("Date of last transaction")
|
||||
.copied();
|
||||
let price_idx = state.feature_name_to_index.get("Last known price").copied();
|
||||
match (year_idx, price_idx) {
|
||||
(Some(yi), Some(pi)) => {
|
||||
let mut points: Vec<PricePoint> = matching_rows
|
||||
.iter()
|
||||
.filter_map(|&row| {
|
||||
let year = feature_data[row * num_features + yi];
|
||||
let price = feature_data[row * num_features + pi];
|
||||
if year.is_finite() && price.is_finite() {
|
||||
Some(PricePoint { year, price })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// Cap at 5000 points by evenly sampling
|
||||
if points.len() > 5000 {
|
||||
let step = points.len() as f64 / 5000.0;
|
||||
points = (0..5000)
|
||||
.map(|i| {
|
||||
let idx = (i as f64 * step) as usize;
|
||||
PricePoint {
|
||||
year: points[idx].year,
|
||||
price: points[idx].price,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
points
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
};
|
||||
let price_history = stats::extract_price_history(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
num_features,
|
||||
&state.feature_name_to_index,
|
||||
);
|
||||
|
||||
let mut numeric_features = Vec::new();
|
||||
let mut enum_features_out = Vec::new();
|
||||
|
||||
for (feature_index, feature_name) in state.data.feature_names.iter().enumerate() {
|
||||
if fields_specified && !field_set.contains(feature_name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(enum_values) = state.data.enum_values.get(&feature_index) {
|
||||
// Enum feature: count occurrences of each value
|
||||
let mut value_counts = vec![0u64; enum_values.len()];
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
let idx = value as usize;
|
||||
if idx < value_counts.len() {
|
||||
value_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let counts: HashMap<String, u64> = value_counts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, &count)| count > 0)
|
||||
.map(|(idx, &count)| (enum_values[idx].clone(), count))
|
||||
.collect();
|
||||
|
||||
if !counts.is_empty() {
|
||||
enum_features_out.push(EnumFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
counts,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Numeric feature: compute stats and histogram
|
||||
let global_hist = &state.data.feature_stats[feature_index].histogram;
|
||||
let p1 = global_hist.p1;
|
||||
let p99 = global_hist.p99;
|
||||
let num_bins = global_hist.counts.len();
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut bins = vec![0u64; num_bins];
|
||||
|
||||
let middle_bins = num_bins.saturating_sub(2);
|
||||
let middle_width = if middle_bins > 0 && p99 > p1 {
|
||||
(p99 - p1) / middle_bins as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
count += 1;
|
||||
if value < min_value {
|
||||
min_value = value;
|
||||
}
|
||||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value as f64;
|
||||
|
||||
let bin = if value < p1 {
|
||||
0
|
||||
} else if value >= p99 {
|
||||
num_bins - 1
|
||||
} else if middle_width > 0.0 {
|
||||
let middle_bin = ((value - p1) / middle_width) as usize;
|
||||
(1 + middle_bin).min(num_bins - 2)
|
||||
} else {
|
||||
num_bins / 2
|
||||
};
|
||||
bins[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
numeric_features.push(NumericFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
count,
|
||||
min: min_value as f64,
|
||||
max: max_value as f64,
|
||||
mean: sum / count as f64,
|
||||
histogram: HistogramStats {
|
||||
min: global_hist.min as f64,
|
||||
max: global_hist.max as f64,
|
||||
p1: p1 as f64,
|
||||
p99: p99 as f64,
|
||||
counts: bins,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
&state.data.feature_names,
|
||||
num_features,
|
||||
&state.data.enum_values,
|
||||
&state.data.feature_stats,
|
||||
fields_specified,
|
||||
&field_set,
|
||||
);
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(
|
||||
|
|
|
|||
|
|
@ -8,8 +8,11 @@ use serde::{Deserialize, Serialize};
|
|||
use serde_json::{Map, Value};
|
||||
use tracing::info;
|
||||
|
||||
use crate::aggregation::Aggregator;
|
||||
use crate::consts::MAX_CELLS_PER_REQUEST;
|
||||
use crate::parsing::{bounds_intersect, parse_bounds, parse_filters, row_passes_filters};
|
||||
use crate::parsing::{
|
||||
bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -27,68 +30,31 @@ pub struct PostcodeParams {
|
|||
fields: Option<String>,
|
||||
}
|
||||
|
||||
/// Per-postcode accumulator for aggregating features.
|
||||
struct PostcodeAgg {
|
||||
count: u32,
|
||||
mins: Box<[f32]>,
|
||||
maxs: Box<[f32]>,
|
||||
sums: Box<[f64]>,
|
||||
feat_counts: Box<[u32]>,
|
||||
}
|
||||
|
||||
impl PostcodeAgg {
|
||||
fn new(num_features: usize) -> Self {
|
||||
PostcodeAgg {
|
||||
count: 0,
|
||||
mins: vec![f32::INFINITY; num_features].into_boxed_slice(),
|
||||
maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(),
|
||||
sums: vec![0.0f64; num_features].into_boxed_slice(),
|
||||
feat_counts: vec![0u32; num_features].into_boxed_slice(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
for (feat_index, &value) in row_slice.iter().enumerate() {
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn add_row_selective(
|
||||
&mut self,
|
||||
feature_data: &[f32],
|
||||
row: usize,
|
||||
num_features: usize,
|
||||
indices: &[usize],
|
||||
) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
for &feat_index in indices {
|
||||
let value = feature_data[base + feat_index];
|
||||
if value.is_finite() {
|
||||
if value < self.mins[feat_index] {
|
||||
self.mins[feat_index] = value;
|
||||
}
|
||||
if value > self.maxs[feat_index] {
|
||||
self.maxs[feat_index] = value;
|
||||
}
|
||||
self.sums[feat_index] += value as f64;
|
||||
self.feat_counts[feat_index] += 1;
|
||||
}
|
||||
}
|
||||
/// Build a GeoJSON geometry object from postcode polygon rings.
|
||||
/// Returns Polygon for 1 ring, MultiPolygon for 2+ rings.
|
||||
fn build_postcode_geometry(rings: &[Vec<[f32; 2]>]) -> Value {
|
||||
if rings.len() == 1 {
|
||||
let coords: Vec<Value> = rings[0]
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![Value::from(*lon as f64), Value::from(*lat as f64)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "Polygon", "coordinates": [coords] })
|
||||
} else {
|
||||
let polys: Vec<Value> = rings
|
||||
.iter()
|
||||
.map(|ring| {
|
||||
let coords: Vec<Value> = ring
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![Value::from(*lon as f64), Value::from(*lat as f64)])
|
||||
})
|
||||
.collect();
|
||||
Value::Array(vec![Value::Array(coords)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "MultiPolygon", "coordinates": polys })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -96,12 +62,7 @@ pub async fn get_postcodes(
|
|||
state: Arc<AppState>,
|
||||
Query(params): Query<PostcodeParams>,
|
||||
) -> Result<Json<PostcodesResponse>, (StatusCode, String)> {
|
||||
let bounds_str = params.bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
|
||||
let (south, west, north, east) = parse_bounds(&bounds_str)?;
|
||||
let (south, west, north, east) = require_bounds(params.bounds)?;
|
||||
|
||||
let filters_str = params.filters.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
|
|
@ -111,22 +72,7 @@ pub async fn get_postcodes(
|
|||
);
|
||||
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
|
||||
|
||||
// Parse optional `fields` param into feature indices
|
||||
let field_indices: Option<Vec<usize>> = params.fields.as_ref().map(|fields_str| {
|
||||
if fields_str.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
fields_str
|
||||
.split(',')
|
||||
.filter_map(|name| {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
state.feature_name_to_index.get(name).copied()
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index);
|
||||
|
||||
let response = tokio::task::spawn_blocking(move || -> Result<PostcodesResponse, String> {
|
||||
let postcode_data = &state.postcode_data;
|
||||
|
|
@ -168,11 +114,11 @@ pub async fn get_postcodes(
|
|||
|
||||
// Aggregate for each postcode that has properties in bounds
|
||||
// (polygon intersection check happens later when building response)
|
||||
let mut postcode_aggs: FxHashMap<usize, PostcodeAgg> = FxHashMap::default();
|
||||
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
|
||||
for (&pc_idx, rows) in &postcode_rows {
|
||||
let agg = postcode_aggs
|
||||
.entry(pc_idx)
|
||||
.or_insert_with(|| PostcodeAgg::new(num_features));
|
||||
.or_insert_with(|| Aggregator::new(num_features));
|
||||
for &row in rows {
|
||||
if has_selective {
|
||||
agg.add_row_selective(feature_data, row, num_features, sel_indices);
|
||||
|
|
@ -222,42 +168,7 @@ pub async fn get_postcodes(
|
|||
continue;
|
||||
}
|
||||
|
||||
// Build GeoJSON geometry: Polygon (1 ring) or MultiPolygon (2+ rings)
|
||||
let geometry = if rings.len() == 1 {
|
||||
let coords: Vec<Value> = rings[0]
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![Value::from(*lon as f64), Value::from(*lat as f64)])
|
||||
})
|
||||
.collect();
|
||||
let mut geo = Map::new();
|
||||
geo.insert("type".into(), Value::String("Polygon".into()));
|
||||
geo.insert(
|
||||
"coordinates".into(),
|
||||
Value::Array(vec![Value::Array(coords)]),
|
||||
);
|
||||
geo
|
||||
} else {
|
||||
let polys: Vec<Value> = rings
|
||||
.iter()
|
||||
.map(|ring| {
|
||||
let coords: Vec<Value> = ring
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![
|
||||
Value::from(*lon as f64),
|
||||
Value::from(*lat as f64),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
Value::Array(vec![Value::Array(coords)])
|
||||
})
|
||||
.collect();
|
||||
let mut geo = Map::new();
|
||||
geo.insert("type".into(), Value::String("MultiPolygon".into()));
|
||||
geo.insert("coordinates".into(), Value::Array(polys));
|
||||
geo
|
||||
};
|
||||
let geometry = build_postcode_geometry(rings);
|
||||
|
||||
// Build properties
|
||||
let centroid = postcode_data.centroids[pc_idx];
|
||||
|
|
@ -300,7 +211,7 @@ pub async fn get_postcodes(
|
|||
// Build GeoJSON Feature
|
||||
let mut feature = Map::new();
|
||||
feature.insert("type".into(), Value::String("Feature".into()));
|
||||
feature.insert("geometry".into(), Value::Object(geometry));
|
||||
feature.insert("geometry".into(), geometry);
|
||||
feature.insert("properties".into(), Value::Object(props));
|
||||
|
||||
features.push(feature);
|
||||
|
|
@ -353,31 +264,7 @@ pub async fn get_postcode_lookup(
|
|||
if let Some(&idx) = postcode_data.postcode_to_idx.get(&normalized) {
|
||||
let (lat, lon) = postcode_data.centroids[idx];
|
||||
let rings = &postcode_data.polygons[idx];
|
||||
|
||||
// Build GeoJSON geometry
|
||||
let geometry = if rings.len() == 1 {
|
||||
let coords: Vec<Value> = rings[0]
|
||||
.iter()
|
||||
.map(|[lo, la]| {
|
||||
Value::Array(vec![Value::from(*lo as f64), Value::from(*la as f64)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "Polygon", "coordinates": [coords] })
|
||||
} else {
|
||||
let polys: Vec<Value> = rings
|
||||
.iter()
|
||||
.map(|ring| {
|
||||
let coords: Vec<Value> = ring
|
||||
.iter()
|
||||
.map(|[lo, la]| {
|
||||
Value::Array(vec![Value::from(*lo as f64), Value::from(*la as f64)])
|
||||
})
|
||||
.collect();
|
||||
Value::Array(vec![Value::Array(coords)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "MultiPolygon", "coordinates": polys })
|
||||
};
|
||||
let geometry = build_postcode_geometry(rings);
|
||||
|
||||
info!(postcode = %normalized, "GET /api/postcode/{postcode}");
|
||||
Ok(Json(serde_json::json!({
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ use rustc_hash::FxHashMap;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{
|
||||
DEFAULT_PROPERTIES_LIMIT, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN,
|
||||
MAX_PROPERTIES_LIMIT,
|
||||
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, MAX_PROPERTIES_LIMIT};
|
||||
use crate::parsing::{
|
||||
cell_for_row, h3_cell_bounds, needs_parent, parse_filters, row_passes_filters,
|
||||
validate_h3_resolution,
|
||||
};
|
||||
use crate::parsing::{h3_cell_bounds, parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
|
@ -103,19 +103,8 @@ pub async fn get_hexagon_properties(
|
|||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
validate_h3_resolution(resolution)?;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
|
|
@ -130,7 +119,7 @@ pub async fn get_hexagon_properties(
|
|||
let precomputed = &state.h3_cells;
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let need_parent = resolution < H3_PRECOMPUTE_MAX;
|
||||
let need_parent = needs_parent(resolution);
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let feature_names = &state.data.feature_names;
|
||||
|
|
@ -139,24 +128,12 @@ pub async fn get_hexagon_properties(
|
|||
|
||||
let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(cell, 0.001);
|
||||
|
||||
let cell_for_row = |row: usize| -> u64 {
|
||||
let max_cell = precomputed[row];
|
||||
if !need_parent || max_cell == 0 {
|
||||
return max_cell;
|
||||
}
|
||||
h3o::CellIndex::try_from(max_cell)
|
||||
.ok()
|
||||
.and_then(|ci| ci.parent(h3_res))
|
||||
.map(u64::from)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let mut matching_rows: Vec<usize> = Vec::new();
|
||||
state
|
||||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if cell_for_row(row) == cell_u64
|
||||
if cell_for_row(row, precomputed, h3_res, need_parent) == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
|
|||
163
server-rs/src/routes/stats.rs
Normal file
163
server-rs/src/routes/stats.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use crate::consts::MAX_PRICE_HISTORY_POINTS;
|
||||
use crate::data::FeatureStats;
|
||||
|
||||
use super::hexagon_stats::{EnumFeatureStats, HistogramStats, NumericFeatureStats, PricePoint};
|
||||
|
||||
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
|
||||
pub fn extract_price_history(
|
||||
matching_rows: &[usize],
|
||||
feature_data: &[f32],
|
||||
num_features: usize,
|
||||
feature_name_to_index: &FxHashMap<String, usize>,
|
||||
) -> Vec<PricePoint> {
|
||||
let year_idx = feature_name_to_index
|
||||
.get("Date of last transaction")
|
||||
.copied();
|
||||
let price_idx = feature_name_to_index.get("Last known price").copied();
|
||||
match (year_idx, price_idx) {
|
||||
(Some(yi), Some(pi)) => {
|
||||
let mut points: Vec<PricePoint> = matching_rows
|
||||
.iter()
|
||||
.filter_map(|&row| {
|
||||
let year = feature_data[row * num_features + yi];
|
||||
let price = feature_data[row * num_features + pi];
|
||||
if year.is_finite() && price.is_finite() {
|
||||
Some(PricePoint { year, price })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
if points.len() > MAX_PRICE_HISTORY_POINTS {
|
||||
let step = points.len() as f64 / MAX_PRICE_HISTORY_POINTS as f64;
|
||||
points = (0..MAX_PRICE_HISTORY_POINTS)
|
||||
.map(|i| {
|
||||
let idx = (i as f64 * step) as usize;
|
||||
PricePoint {
|
||||
year: points[idx].year,
|
||||
price: points[idx].price,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
points
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute per-feature stats (numeric histograms + enum counts) for the given rows.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn compute_feature_stats(
|
||||
matching_rows: &[usize],
|
||||
feature_data: &[f32],
|
||||
feature_names: &[String],
|
||||
num_features: usize,
|
||||
enum_values: &FxHashMap<usize, Vec<String>>,
|
||||
feature_stats_data: &[FeatureStats],
|
||||
fields_specified: bool,
|
||||
field_set: &HashSet<String>,
|
||||
) -> (Vec<NumericFeatureStats>, Vec<EnumFeatureStats>) {
|
||||
let mut numeric_features = Vec::new();
|
||||
let mut enum_features_out = Vec::new();
|
||||
|
||||
for (feature_index, feature_name) in feature_names.iter().enumerate() {
|
||||
if fields_specified && !field_set.contains(feature_name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(ev) = enum_values.get(&feature_index) {
|
||||
let mut value_counts = vec![0u64; ev.len()];
|
||||
for &row in matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
let idx = value as usize;
|
||||
if idx < value_counts.len() {
|
||||
value_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let counts: HashMap<String, u64> = value_counts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, &count)| count > 0)
|
||||
.map(|(idx, &count)| (ev[idx].clone(), count))
|
||||
.collect();
|
||||
|
||||
if !counts.is_empty() {
|
||||
enum_features_out.push(EnumFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
counts,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
let global_hist = &feature_stats_data[feature_index].histogram;
|
||||
let p1 = global_hist.p1;
|
||||
let p99 = global_hist.p99;
|
||||
let num_bins = global_hist.counts.len();
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut bins = vec![0u64; num_bins];
|
||||
|
||||
let middle_bins = num_bins.saturating_sub(2);
|
||||
let middle_width = if middle_bins > 0 && p99 > p1 {
|
||||
(p99 - p1) / middle_bins as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
for &row in matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
count += 1;
|
||||
if value < min_value {
|
||||
min_value = value;
|
||||
}
|
||||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value as f64;
|
||||
|
||||
let bin = if value < p1 {
|
||||
0
|
||||
} else if value >= p99 {
|
||||
num_bins - 1
|
||||
} else if middle_width > 0.0 {
|
||||
let middle_bin = ((value - p1) / middle_width) as usize;
|
||||
(1 + middle_bin).min(num_bins - 2)
|
||||
} else {
|
||||
num_bins / 2
|
||||
};
|
||||
bins[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
numeric_features.push(NumericFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
count,
|
||||
min: min_value as f64,
|
||||
max: max_value as f64,
|
||||
mean: sum / count as f64,
|
||||
histogram: HistogramStats {
|
||||
min: global_hist.min as f64,
|
||||
max: global_hist.max as f64,
|
||||
p1: p1 as f64,
|
||||
p99: p99 as f64,
|
||||
counts: bins,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(numeric_features, enum_features_out)
|
||||
}
|
||||
|
|
@ -31,7 +31,7 @@ pub struct AppState {
|
|||
pub features_response: FeaturesResponse,
|
||||
/// URL of the screenshot service (e.g. http://screenshot:8002)
|
||||
pub screenshot_url: String,
|
||||
/// Public-facing URL for absolute og:image URLs (e.g. https://narrowit.schmelczer.dev)
|
||||
/// Public-facing URL for absolute og:image URLs (e.g. https://perfectpostcodes.schmelczer.dev)
|
||||
pub public_url: String,
|
||||
/// Contents of index.html read at startup, used for crawler OG injection
|
||||
pub index_html: Option<String>,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue