diff --git a/CLAUDE.md b/CLAUDE.md index b3744e8..1b20795 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -302,7 +302,7 @@ Follow these conventions in all Rust code: ``` 4. **JSON serialization**: Use `serde_json` with `#[derive(Serialize)]` structs, not manual string building 5. **Precompute at startup**: For static/rarely-changing responses, compute once at startup and store in `AppState` -6. **Unique placeholders**: When injecting content into HTML, use distinctive markers like `__NARROWIT_OG_TAGS__` that won't accidentally match other content +6. **Unique placeholders**: When injecting content into HTML, use distinctive markers like `__PERFECT_POSTCODES_OG_TAGS__` that won't accidentally match other content ## Key Implementation Details @@ -317,7 +317,7 @@ Follow these conventions in all Rust code: - **Server-side AABB filtering**: Both `/api/hexagons` and `/api/postcodes` filter results by bounding-box intersection with query bounds. Hexagons use `h3_cell_bounds()` (h3o returns degrees, not radians). Postcodes compute polygon AABB from vertices. See `bounds_intersect()` in `parsing/bounds.rs`. - **GridIndex returns slightly more than requested**: The 0.01° grid cells mean properties up to ~1km outside the viewport may be returned. The AABB filter in the route handlers catches these extras. - **POI proximity**: Uses 0.05° grid (~5km cells) to reduce candidates before haversine distance check -- **OG tag injection**: Uses `` placeholder in HTML, replaced at runtime by middleware +- **OG tag injection**: Uses `` placeholder in HTML, replaced at runtime by middleware ## Rust Performance Patterns (server-rs) diff --git a/Makefile.data b/Makefile.data new file mode 100644 index 0000000..d300565 --- /dev/null +++ b/Makefile.data @@ -0,0 +1,235 @@ +# Data pipeline — download sources and build wide.parquet +# +# Usage: +# make -f Makefile.data prepare # Build wide.parquet (+ all deps) +# make -f Makefile.data tiles # Download UK map tiles +# make -f Makefile.data download-pois # Download a single dataset +# make -f Makefile.data help # List all targets +# +# Or include from the main Makefile and use targets directly. + +SHELL := /bin/bash +.DELETE_ON_ERROR: + +DATA_DIR := /bulk/property-data + +# ── Output files ────────────────────────────────────────────────────────────── + +TILES := $(DATA_DIR)/uk.pmtiles +ARCGIS := $(DATA_DIR)/arcgis_data.parquet +PRICE_PAID := $(DATA_DIR)/price-paid-complete.parquet +IOD := $(DATA_DIR)/IoD2025_Scores.parquet +POIS_RAW := $(DATA_DIR)/uk_pois.parquet +POIS_FILTERED := $(DATA_DIR)/filtered_uk_pois.parquet +POI_PROXIMITY := $(DATA_DIR)/poi_proximity.parquet +EPC_PP := $(DATA_DIR)/epc_pp.parquet +WIDE := $(DATA_DIR)/wide.parquet +EPC := $(DATA_DIR)/certificates.csv +JT_BANK := $(DATA_DIR)/journey_times_bank.parquet +JT_FITZROVIA := $(DATA_DIR)/journey_times_fitzrovia.parquet +ETHNICITY := $(DATA_DIR)/ethnicity_by_la.parquet +CRIME_DIR := $(DATA_DIR)/crime +CRIME := $(DATA_DIR)/crime_by_lsoa.parquet +NOISE := $(DATA_DIR)/road_noise.parquet +OFSTED := $(DATA_DIR)/ofsted.parquet +NAPTAN := $(DATA_DIR)/naptan.parquet +BROADBAND := $(DATA_DIR)/broadband.parquet +SCHOOL_PROX := $(DATA_DIR)/school_proximity.parquet +GEOSURE_DIR := $(DATA_DIR)/geosure +GEOSURE := $(DATA_DIR)/geosure.parquet +INSPIRE_DIR := $(DATA_DIR)/inspire +OA_BOUNDARIES := $(DATA_DIR)/oa_boundaries.gpkg +UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet +PC_BOUNDARIES := $(DATA_DIR)/new_postcode_boundaries + +# Sentinel files for directory targets (Make doesn't track directories well) +GEOSURE_STAMP := $(GEOSURE_DIR)/.done +INSPIRE_STAMP := $(INSPIRE_DIR)/.done + +MANUAL_DATA := $(DATA_DIR)/manual_data +PMTILES_VERSION := 1.22.3 +PMTILES_BIN := $(DATA_DIR)/pmtiles + +# ── Phony aliases ───────────────────────────────────────────────────────────── + +.PHONY: prepare tiles \ + download-arcgis download-price-paid download-deprivation download-ethnicity \ + download-naptan download-pois download-ofsted download-broadband \ + download-postcodes download-geosure download-noise download-inspire \ + download-oa-boundaries download-uprn-lookup \ + transform-pois transform-epc-pp transform-crime transform-poi-proximity \ + transform-school-proximity transform-geosure transform-postcode-boundaries \ + journey-times + +prepare: $(WIDE) +tiles: $(TILES) +download-arcgis: $(ARCGIS) +download-price-paid: $(PRICE_PAID) +download-deprivation: $(IOD) +download-ethnicity: $(ETHNICITY) +download-naptan: $(NAPTAN) +download-pois: $(POIS_RAW) +download-ofsted: $(OFSTED) +download-broadband: $(BROADBAND) +download-postcodes: $(POSTCODES) +download-geosure: $(GEOSURE_STAMP) +download-noise: $(NOISE) +download-inspire: $(INSPIRE_STAMP) +download-oa-boundaries: $(OA_BOUNDARIES) +download-uprn-lookup: $(UPRN_LOOKUP) +transform-pois: $(POIS_FILTERED) +transform-epc-pp: $(EPC_PP) +transform-crime: $(CRIME) +transform-poi-proximity: $(POI_PROXIMITY) +transform-school-proximity: $(SCHOOL_PROX) +transform-geosure: $(GEOSURE) +transform-postcode-boundaries: $(PC_BOUNDARIES) + +# ── Downloads ───────────────────────────────────────────────────────────────── + +$(TILES): + @echo "Downloading UK PMTiles (~1.5GB)..." + @echo "This extracts UK tiles from the Protomaps planet file." + @if [ ! -f "$(PMTILES_BIN)" ]; then \ + echo "Downloading pmtiles CLI v$(PMTILES_VERSION)..."; \ + curl -sL "https://github.com/protomaps/go-pmtiles/releases/download/v$(PMTILES_VERSION)/go-pmtiles_$(PMTILES_VERSION)_Linux_x86_64.tar.gz" \ + | tar -xz -C "$(DATA_DIR)" pmtiles; \ + chmod +x "$(PMTILES_BIN)"; \ + fi + "$(PMTILES_BIN)" extract https://build.protomaps.com/20260201.pmtiles $@ --bbox=-10.5,49.5,2.5,61 + +# EPC requires manual registration — fail with instructions +$(EPC): + @echo "" + @echo "=== EPC dataset not found ===" + @echo "The EPC certificates file is required: $@" + @echo "" + @echo "To obtain it, register at https://epc.opendatacommunities.org/login" + @echo "" + @exit 1 + +$(ARCGIS): + uv run python -m pipeline.download.arcgis --output $@ + +$(PRICE_PAID): + uv run python -m pipeline.download.price_paid --output $@ + +$(IOD): + uv run python -m pipeline.download.deprivation_data --output $@ + +$(ETHNICITY): + uv run python -m pipeline.download.ethnicity --output $@ + +$(NAPTAN): + uv run python -m pipeline.download.naptan --output $@ + +$(POIS_RAW): + uv run python -m pipeline.download.pois --output $@ + +$(OFSTED): + uv run python -m pipeline.download.ofsted --output $@ + +$(BROADBAND): + uv run python -m pipeline.download.broadband --output $@ + +$(POSTCODES): + uv run python -m pipeline.download.postcodes --output $@ + +$(GEOSURE_STAMP): + uv run python -m pipeline.download.geosure --output $(GEOSURE_DIR) + @touch $@ + +$(NOISE): $(ARCGIS) + uv run python -m pipeline.download.noise --arcgis $(ARCGIS) --output $@ + +$(INSPIRE_STAMP): + uv run python -m pipeline.download.inspire --output $(INSPIRE_DIR) + @touch $@ + +$(OA_BOUNDARIES): + uv run python -m pipeline.download.oa_boundaries --output $@ + +$(UPRN_LOOKUP): + uv run python -m pipeline.download.uprn_lookup --output $@ + +# ── Journey times (requires TFL_API_KEY) ────────────────────────────────────── + +$(JT_BANK): + @if [ -f "$(MANUAL_DATA)/journey_times_bank.parquet" ]; then \ + echo "Copying journey_times_bank.parquet from manual_data/"; \ + cp "$(MANUAL_DATA)/journey_times_bank.parquet" $@; \ + else \ + echo ""; \ + echo "=== TFL journey times (bank) not found ==="; \ + echo "Either place the file in $(MANUAL_DATA)/journey_times_bank.parquet"; \ + echo "or register for a TFL API key at https://api-portal.tfl.gov.uk/signin"; \ + echo "and run: TFL_API_KEY=... make -f Makefile.data journey-times DEST=bank"; \ + echo ""; \ + exit 1; \ + fi + +$(JT_FITZROVIA): + @if [ -f "$(MANUAL_DATA)/journey_times_fitzrovia.parquet" ]; then \ + echo "Copying journey_times_fitzrovia.parquet from manual_data/"; \ + cp "$(MANUAL_DATA)/journey_times_fitzrovia.parquet" $@; \ + else \ + echo ""; \ + echo "=== TFL journey times (fitzrovia) not found ==="; \ + echo "Either place the file in $(MANUAL_DATA)/journey_times_fitzrovia.parquet"; \ + echo "or register for a TFL API key at https://api-portal.tfl.gov.uk/signin"; \ + echo "and run: TFL_API_KEY=... make -f Makefile.data journey-times DEST=fitzrovia"; \ + echo ""; \ + exit 1; \ + fi + +journey-times: $(ARCGIS) +ifndef DEST + $(error DEST required — e.g. make journey-times DEST=bank) +endif + uv run python -m pipeline.journey_times --destination $(DEST) --output-dir $(DATA_DIR) --postcodes $(ARCGIS) + +# ── Transforms ──────────────────────────────────────────────────────────────── + +$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) + uv run python -m pipeline.transform.transform_poi --input $(POIS_RAW) --naptan $(NAPTAN) --output $@ + +$(EPC_PP): $(PRICE_PAID) $(EPC) + uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@ + +$(CRIME): + uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@ + +$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) + uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --output $@ + +$(SCHOOL_PROX): $(OFSTED) $(ARCGIS) + uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@ + +$(GEOSURE): $(GEOSURE_STAMP) $(ARCGIS) + uv run python -m pipeline.transform.transform_geosure --geosure $(GEOSURE_DIR) --arcgis $(ARCGIS) --output $@ + +$(PC_BOUNDARIES): $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP) + uv run python -m pipeline.transform.postcode_boundaries \ + --uprn $(UPRN_LOOKUP) \ + --oa-boundaries $(OA_BOUNDARIES) \ + --inspire $(INSPIRE_DIR) \ + --output $@ + +# ── Final merge ─────────────────────────────────────────────────────────────── + +$(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) \ + $(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE) + uv run python -m pipeline.transform.merge \ + --epc-pp $(EPC_PP) \ + --arcgis $(ARCGIS) \ + --iod $(IOD) \ + --poi-proximity $(POI_PROXIMITY) \ + --journey-times-bank $(JT_BANK) \ + --journey-times-fitzrovia $(JT_FITZROVIA) \ + --ethnicity $(ETHNICITY) \ + --crime $(CRIME) \ + --noise $(NOISE) \ + --school-proximity $(SCHOOL_PROX) \ + --broadband $(BROADBAND) \ + --geosure $(GEOSURE) \ + --output $@ diff --git a/README.md b/README.md index a0397e9..eecb882 100644 --- a/README.md +++ b/README.md @@ -34,20 +34,40 @@ epc oopt out https://www.gov.uk/guidance/energy-performance-certificates-opt-out + + +We mapped every neighbourhood in England. You're welcome. + +Harness our supercharged data to find your perfect postcode + + +Set the interactive filters to match your preferences and watch the hidden gems get uncovered - instantly. Only you know your preferences. We will give you all the data that exists, aggregated into dozens of personalised sliders for you to compile the list of perfect postcodes. + + + +Now imagine this, but with 43 different filters across noise levels, environment risk, nearby amenities, demographics, socioeconomics, crime rate, transport link, and properties. (show the filter types with small cards). + + +We strongly believe that the smart way to buy a property is by first understanding your expectations. + + + +We give you the best-in-class tools to be intentional about the trade-offs you make in one of the most important decisions of your life. + + + + + + + You may buy a box of cereal you’ve never tried because it’s 20% off. Your future home is not a box of cereal. Don’t let a seemingly good deal that just popped up turn into life-long regret because you got a discount but not on the home you needed. We help you reverse the equation and allow you to be intentional in your search. Don’t wait for good deals to come your way, if you can go out and find them yourself. -You can’t get a complete picture of the property market by looking at current listings. To understand the landscape, you have to look at historical trends. +There are too few properties listed at any time to give you a full and nuanced picture of the entire property landscape but there’re too many to go through one by one and evaluate them all. You can’t get a complete picture of the property market by looking at current listings. To understand the landscape, you have to look at historical trends. Don’t let the market sway you, anchor your expectations based on the choices millions of other buyers who’ve been in the same situation as you are now. -We give you the tools to be intentional about the trade-offs you make in one of the most important decisions of your life. - -Don’t let the market sway you, anchor your expectations based on the choices millions of other buyers who’ve been in the same situation as you are now. - -There are too few properties listed at any time to give you a full and nuanced picture of the entire property landscape but there’re too many to go through one by one and evaluate them all. We all care about different things in our homes and living environments. Some of us are weary of noise and would like to avoid living next to a loud airfield as much as possible. And some of us are avid plane spotters. -Only you know your preferences. We will give you all the data that exists, aggregated into dozens of personalised sliders for you to compile the list of perfect postcodes. We will help you find the best places to live within your budget regardless if there’s a property listed there right now. The best things come to those who’re patient. We will justify your patience. But we will also show you if your expectations are impossible to meet. I’d much rather be told upfront then spend months of my life looking for something that can’t possibly exist. @@ -58,18 +78,13 @@ We give you all the data and tools to become an Well-informed Buyer through the --- -- scraping - fix frontend - map hexagons - - dragging -- account management - stripe - update texts -- fix plausible - move data to raid - extract all user-facing texts into a yaml file for easy editing -- register domain - register for email @@ -86,33 +101,12 @@ FAQ: ## outstadning prompts -Add licensing to the app. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too. While doing this, protect the server against DOS-ing. - +Add licensing to the app. By default, anonymous users can use the map but only in central london. if they try zooming out, the server refuses to provide data and the users will be prompted to buy a lifetime license to continue (or zoom back in). Just before buying a license, they have to register by providing their email address and password, then they need to complete the stripe check out workflow. Implement the full pocketbase/server/frontend integration. For admins, give an option to generate an invite link, opening which prompts you to register and gives you a free license forever. Have a cool animation with party poppers on the successful acquiring of a license. For non-admin users, allow inviting friends for 30% off the price. Also add a support page that shows my email address, and add a FAQ on the same page too. - - the area stastics are missing for postcodes, they only work for hexagons -- in the mobile view, move the property density and previewing colour spectrum to the bottom half of the screen. -- make the no active filters have less padding on phone - add blue/green rollout -- rename OgImageQuery to ScreenshotQuery -- make the eye and plus icons and their touch targets twice the size - - - - -## name ideas - - -perfect postcodes - -golden postcodes - -calculated move - -the spec - -geologic diff --git a/docker-compose.yml b/docker-compose.yml index e30c8ba..eda4612 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: screenshot: build: /volumes/syncthing/Projects/property-map/screenshot environment: - NARROWIT_URL: http://frontend:3001 + APP_URL: http://frontend:3001 CACHE_DIR: /cache volumes: - screenshot-cache:/cache diff --git a/frontend/src/components/faq/FAQPage.tsx b/frontend/src/components/faq/FAQPage.tsx index 5b57fcf..34b67de 100644 --- a/frontend/src/components/faq/FAQPage.tsx +++ b/frontend/src/components/faq/FAQPage.tsx @@ -10,7 +10,7 @@ const FAQ_ITEMS: FAQItem[] = [ { question: 'What is this application?', answer: - 'Narrowit is an interactive map that visualises property-level data across England and Wales. It combines Land Registry sale prices, EPC energy certificates, TfL journey times, deprivation indices, crime statistics, broadband speeds, school ratings, road noise levels, ethnicity demographics, and OpenStreetMap points of interest into a single explorable view.', + 'Perfect Postcodes is an interactive map that visualises property-level data across England and Wales. It combines Land Registry sale prices, EPC energy certificates, TfL journey times, deprivation indices, crime statistics, broadband speeds, school ratings, road noise levels, ethnicity demographics, and OpenStreetMap points of interest into a single explorable view.', }, { question: 'Where does the data come from?', @@ -101,7 +101,7 @@ export default function FAQPage() { Frequently Asked Questions

- Common questions about how Narrowit works, where the data comes from, and how to use the + Common questions about how Perfect Postcodes works, where the data comes from, and how to use the map.

diff --git a/frontend/src/components/home/HomePage.tsx b/frontend/src/components/home/HomePage.tsx index 219c26c..9474261 100644 --- a/frontend/src/components/home/HomePage.tsx +++ b/frontend/src/components/home/HomePage.tsx @@ -57,7 +57,7 @@ export default function HomePage({

Set the commute, budget, school rating, noise level, and crime threshold you'll - accept. Narrowit shows you every area that qualifies — instantly. + accept. Perfect Postcodes shows you every area that qualifies — instantly.

+ {expanded && ( + <> + {error ? ( +
+ Failed to generate summary. +
+ ) : loading ? ( +
+
+
+
+ ) : ( +

{summary}

+ )} + + )} +
+
+ ); +} diff --git a/frontend/src/components/map/AreaPane.tsx b/frontend/src/components/map/AreaPane.tsx index 78225ac..acca3ad 100644 --- a/frontend/src/components/map/AreaPane.tsx +++ b/frontend/src/components/map/AreaPane.tsx @@ -1,4 +1,5 @@ import { useMemo, useState } from 'react'; +import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups'; import type { FeatureFilters, FeatureMeta, @@ -15,13 +16,15 @@ import StackedBarChart from './StackedBarChart'; import StackedEnumChart from './StackedEnumChart'; import PriceHistoryChart from './PriceHistoryChart'; import ExternalSearchLinks from './ExternalSearchLinks'; -import { InfoIcon, CloseIcon, ChevronIcon } from '../ui/icons'; +import { InfoIcon, CloseIcon } from '../ui/icons'; import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader'; -import { LightbulbIcon } from '../ui/icons/LightbulbIcon'; import { IconButton } from '../ui/IconButton'; import { FeatureInfoPopup } from '../ui/FeatureInfoPopup'; import { EmptyState } from '../ui/EmptyState'; import { FeatureLabel } from '../ui/FeatureLabel'; +import AISummaryCard from './AISummaryCard'; +import StreetViewEmbed from './StreetViewEmbed'; +import HistogramLegend from './HistogramLegend'; interface AreaPaneProps { stats: HexagonStatsResponse | null; @@ -60,17 +63,9 @@ export default function AreaPane({ const propertyCount = isPostcode && postcodeData ? postcodeData.properties.count : stats?.count; const featureGroups = useMemo(() => groupFeaturesByCategory(globalFeatures), [globalFeatures]); const [infoFeature, setInfoFeature] = useState(null); - const [collapsedGroups, setCollapsedGroups] = useState>(new Set()); + const [collapsedGroups, toggleGroup] = useCollapsibleGroups(); const [aiSummaryExpanded, setAiSummaryExpanded] = useState(true); - const toggleGroup = (name: string) => - setCollapsedGroups((prev) => { - const next = new Set(prev); - if (next.has(name)) next.delete(name); - else next.add(name); - return next; - }); - const numericByName = useMemo(() => { if (!stats) return new Map(); return new Map(stats.numeric_features.map((feature) => [feature.name, feature])); @@ -138,78 +133,18 @@ export default function AreaPane({ )}
- {/* AI Summary Card */} - {(aiSummary || aiSummaryLoading || aiSummaryError) && ( -
-
- - {aiSummaryExpanded && ( - <> - {aiSummaryError ? ( -
- Failed to generate summary. -
- ) : aiSummaryLoading ? ( -
-
-
-
- ) : ( -

- {aiSummary} -

- )} - - )} -
-
- )} + setAiSummaryExpanded(!aiSummaryExpanded)} + /> {loading && !stats ? ( ) : stats ? (
- {/* Histogram color legend */} -
-
-
-
- - Teal bars{' '} - show the distribution in this selected area - -
-
-
- - Gray bars{' '} - show the overall distribution across all areas - -
-
-
- - - Dashed line - {' '} - indicates the global average - -
-
-
+ {featureGroups.map((group) => { const hasData = group.features.some( (feature) => numericByName.has(feature.name) || enumByName.has(feature.name) @@ -460,25 +395,7 @@ export default function AreaPane({
); })} - {/* Google Street View */} - {hexagonLocation && ( -
-
- Street View -
-
-
-