diff --git a/Dockerfile b/Dockerfile index 7197f0b..bea6458 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,9 +22,10 @@ COPY --from=frontend /app/frontend/dist ./frontend/dist/ COPY property-data/wide.parquet ./data/ COPY property-data/filtered_uk_pois.parquet ./data/ +COPY property-data/places.parquet ./data/ COPY property-data/uk.pmtiles ./data/ COPY manual-data/postcode_boundaries ./data/postcode_boundaries/ EXPOSE 8001 ENTRYPOINT ["./property-map-server"] -CMD ["--data", "/app/data/wide.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries"] +CMD ["--data", "/app/data/wide.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries"] diff --git a/Makefile.data b/Makefile.data index 0e12cd4..395b184 100644 --- a/Makefile.data +++ b/Makefile.data @@ -36,6 +36,7 @@ OFSTED := $(DATA_DIR)/ofsted.parquet NAPTAN := $(DATA_DIR)/naptan.parquet BROADBAND := $(DATA_DIR)/broadband.parquet SCHOOL_PROX := $(DATA_DIR)/school_proximity.parquet +RENTAL := $(DATA_DIR)/rental_prices.parquet GEOSURE_DIR := $(DATA_DIR)/geosure GEOSURE := $(DATA_DIR)/geosure.parquet INSPIRE_DIR := $(DATA_DIR)/inspire @@ -44,6 +45,7 @@ UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet PC_BOUNDARIES := $(MANUAL_DATA)/postcode_boundaries TRANSIT_DIR := $(DATA_DIR)/transit TRANSIT_STAMP := $(TRANSIT_DIR)/.done +GREENSPACE := $(DATA_DIR)/greenspace_water.parquet # Sentinel files for directory targets (Make doesn't track directories well) GEOSURE_STAMP := $(GEOSURE_DIR)/.done @@ -55,9 +57,9 @@ PMTILES_VERSION := 1.22.3 .PHONY: prepare wide tiles \ download-arcgis download-price-paid download-deprivation download-ethnicity \ - download-naptan download-pois download-ofsted download-broadband \ + download-naptan download-pois download-ofsted download-broadband download-rental-prices \ download-postcodes download-geosure download-noise download-inspire \ - download-oa-boundaries download-uprn-lookup download-transit-network \ + download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace \ transform-pois transform-epc-pp transform-crime transform-poi-proximity \ transform-school-proximity transform-geosure transform-postcode-boundaries \ generate-postcode-boundaries \ @@ -76,11 +78,13 @@ download-ofsted: $(OFSTED) download-broadband: $(BROADBAND) download-postcodes: $(POSTCODES) download-geosure: $(GEOSURE_STAMP) +download-rental-prices: $(RENTAL) download-noise: $(NOISE) download-inspire: $(INSPIRE_STAMP) download-oa-boundaries: $(OA_BOUNDARIES) download-uprn-lookup: $(UPRN_LOOKUP) download-transit-network: $(TRANSIT_STAMP) +download-greenspace: $(GREENSPACE) transform-pois: $(POIS_FILTERED) transform-epc-pp: $(EPC_PP) transform-crime: $(CRIME) @@ -159,6 +163,12 @@ $(TRANSIT_STAMP): uv run python -m pipeline.download.transit_network --output $(TRANSIT_DIR) @touch $@ +$(RENTAL): + uv run python -m pipeline.download.rental_prices --output $@ + +$(GREENSPACE): + uv run python -m pipeline.download.greenspace_water --output $@ + # ── Journey times (requires TFL_API_KEY) ────────────────────────────────────── $(JT_BANK): @@ -231,7 +241,7 @@ $(PC_BOUNDARIES): # ── Final merge ─────────────────────────────────────────────────────────────── $(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) \ - $(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE) + $(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE) $(RENTAL) uv run python -m pipeline.transform.merge \ --epc-pp $(EPC_PP) \ --arcgis $(ARCGIS) \ @@ -245,6 +255,7 @@ $(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) --school-proximity $(SCHOOL_PROX) \ --broadband $(BROADBAND) \ --geosure $(GEOSURE) \ + --rental-prices $(RENTAL) \ --output $@ # ── Price estimation (post-merge) ──────────────────────────────────────────── diff --git a/Taskfile.yml b/Taskfile.yml index b02d9ba..1c3df20 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -16,10 +16,10 @@ tasks: cmds: - uv run python -m pipeline.download.map_assets --output frontend/public/assets - download:greenspace: - desc: Extract park/water polygons from OSM PBF for postcode boundary trimming + download:places: + desc: Extract place names from OSM PBF cmds: - - uv run python -m pipeline.download.greenspace_water --output data/greenspace_water.parquet {{.CLI_ARGS}} + - uv run python -m pipeline.download.places --output ./property_data/places.parquet {{.CLI_ARGS}} test: desc: Run all tests (Python and Rust) diff --git a/docker-compose.yml b/docker-compose.yml index 1c85931..7e415e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,12 @@ services: server: image: rust:1.84 + init: true working_dir: /app/server-rs command: > bash -c " cargo install cargo-watch && - cargo watch -x 'run -- --data /app/data/wide.parquet --pois /app/data/filtered_uk_pois.parquet --tiles /app/data/uk.pmtiles --postcodes /app/data/postcode_boundaries' + cargo watch -x 'run -- --data /app/data/wide.parquet --pois /app/data/filtered_uk_pois.parquet --places /app/data/places.parquet --tiles /app/data/uk.pmtiles --postcodes /app/data/postcode_boundaries' " ports: - "8001:8001" @@ -18,7 +19,6 @@ services: - cargo-registry:/usr/local/cargo/registry - cargo-target:/app/server-rs/target - ./property-data:/app/data:ro - environment: POCKETBASE_URL: http://pocketbase:8090 POCKETBASE_ADMIN_EMAIL: ${POCKETBASE_ADMIN_EMAIL:-} @@ -31,6 +31,7 @@ services: condition: service_healthy screenshot: + init: true build: /volumes/syncthing/Projects/property-map/screenshot environment: APP_URL: http://frontend:3001 @@ -54,6 +55,7 @@ services: count: 1 frontend: + init: true image: node:22-slim working_dir: /app/frontend command: > @@ -73,6 +75,7 @@ services: PB_PROXY_TARGET: http://pocketbase:8090 pocketbase: + init: true image: ghcr.io/muchobien/pocketbase:latest ports: - "8090:8090" @@ -88,6 +91,7 @@ services: start_period: 5s r5: + init: true build: ./r5-java ports: - "8004:8003" @@ -96,8 +100,10 @@ services: volumes: - r5-network:/data/network - ./property-data/transit:/data/transit:ro + - ./property-data/transit/raw:/data/transit-raw:ro environment: DATA_DIR: /data/transit + OSM_DIR: /data/transit-raw NETWORK_CACHE_DIR: /data/network healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8003/health"] diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b7c3ade..21a0680 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -191,7 +191,7 @@ export default function App() { initialFilters={urlState.filters || {}} initialViewState={initialViewState} initialPOICategories={urlState.poiCategories || new Set()} - initialTab={urlState.tab || 'pois'} + initialTab={urlState.tab || 'area'} initialLoading={initialLoading} theme={theme} pendingInfoFeature={null} @@ -249,7 +249,7 @@ export default function App() { initialFilters={urlState.filters || {}} initialViewState={initialViewState} initialPOICategories={urlState.poiCategories || new Set()} - initialTab={urlState.tab || 'pois'} + initialTab={urlState.tab || 'area'} initialLoading={initialLoading} theme={theme} pendingInfoFeature={pendingInfoFeature} diff --git a/frontend/src/components/data-sources/DataSources.tsx b/frontend/src/components/data-sources/DataSources.tsx deleted file mode 100644 index 80cd48d..0000000 --- a/frontend/src/components/data-sources/DataSources.tsx +++ /dev/null @@ -1,10 +0,0 @@ -export default function DataSources({ onNavigate }: { onNavigate: () => void }) { - return ( - - ); -} diff --git a/frontend/src/components/data-sources/DataSourcesPage.tsx b/frontend/src/components/data-sources/DataSourcesPage.tsx deleted file mode 100644 index 650656c..0000000 --- a/frontend/src/components/data-sources/DataSourcesPage.tsx +++ /dev/null @@ -1,249 +0,0 @@ -import { useEffect, useState, useRef } from 'react'; - -const DATA_SOURCES = [ - { - id: 'price-paid', - name: 'Price Paid Data', - origin: 'HM Land Registry', - use: 'Complete historical property sale prices for England and Wales. Used for the last known sale price of each property.', - url: 'https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads', - license: 'Open Government Licence v3.0', - }, - { - id: 'epc', - name: 'Energy Performance Certificates (EPC)', - origin: 'Ministry of Housing, Communities & Local Government', - use: 'Domestic Energy Performance Certificates providing floor area, number of rooms, construction age, energy ratings, property type, and built form. Fuzzy-joined with Price Paid records by address within postcode buckets. Property owners can opt out of public disclosure.', - optOutUrl: 'https://www.gov.uk/guidance/energy-performance-certificates-opt-out-of-public-disclosure', - url: 'https://epc.opendatacommunities.org/downloads/domestic', - license: 'Open Government Licence v3.0', - }, - { - id: 'nspl', - name: 'National Statistics Postcode Lookup (NSPL)', - origin: 'ONS / ArcGIS', - use: 'Maps postcodes to latitude/longitude, LSOA, and Output Area codes for geolocation and joining area-level datasets.', - url: 'https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data', - license: 'Open Government Licence v3.0', - }, - { - id: 'iod', - name: 'English Indices of Deprivation 2025', - origin: 'Ministry of Housing, Communities & Local Government', - use: 'Relative deprivation scores for 33,755 LSOAs across domains: Income, Employment, Education, Health, Crime, Living Environment, and sub-domains. Joined to properties via LSOA code.', - url: 'https://www.gov.uk/government/statistics/english-indices-of-deprivation-2025', - license: 'Open Government Licence v3.0', - }, - { - id: 'ethnicity', - name: 'Population by Ethnicity (2021 Census)', - origin: 'ONS', - use: 'Population percentages by ethnic group (Asian, Black, Mixed, White, Other) per Local Authority. Joined via Local Authority District code.', - url: 'https://www.ethnicity-facts-figures.service.gov.uk/uk-population-by-ethnicity/national-and-regional-populations/regional-ethnic-diversity/latest/#download-the-data', - license: 'Open Government Licence v3.0', - }, - { - id: 'crime', - name: 'Street-level Crime Data', - origin: 'data.police.uk', - use: 'Street-level crime data from 2023 to 2025, aggregated into yearly averages by LSOA and crime type (violence, burglary, anti-social behaviour, drugs, vehicle crime, etc.).', - url: 'https://data.police.uk/data/', - license: 'Open Government Licence v3.0', - }, - { - id: 'tfl-journey-times', - name: 'TfL Journey Times', - origin: 'Transport for London', - use: "Journey time calculations from postcodes to central London destinations (Bank, Waterloo, King's Cross, etc.) via public transport and cycling.", - url: 'https://api-portal.tfl.gov.uk/', - license: 'Powered by TfL Open Data', - }, - { - id: 'osm-pois', - name: 'OpenStreetMap POIs', - origin: 'OpenStreetMap contributors / Geofabrik', - use: 'Points of interest extracted from the Great Britain PBF extract. Covers amenities, shops, healthcare, leisure, tourism, and more. Filtered and remapped to friendly category names.', - url: 'https://download.geofabrik.de/europe/great-britain-latest.osm.pbf', - license: 'Open Data Commons Open Database License (ODbL)', - }, - { - id: 'naptan', - name: 'NaPTAN (Public Transport Stops)', - origin: 'Department for Transport', - use: 'National Public Transport Access Nodes providing station and stop locations (rail, bus, metro/tram, ferry, airport), merged into the POI dataset.', - url: 'https://naptan.dft.gov.uk/naptan/schema/2.4/doc/NaPTANSchemaGuide-2.4-v0.57.pdf', - license: 'Open Government Licence v3.0', - }, - { - id: 'noise', - name: 'Defra Noise Mapping', - origin: 'Defra / Environment Agency', - use: 'Strategic noise mapping Round 4 (2022) for road, rail, and airport sources. Lden (day-evening-night 24h weighted average) at 10m grid resolution, modelled at 4m above ground. Sampled at postcode centroids via WCS GeoTIFF tiles.', - url: 'https://environment.data.gov.uk/spatialdata/road-noise-all-metrics-england-round-4/wcs', - license: 'Open Government Licence v3.0', - }, - { - id: 'ofsted', - name: 'Ofsted School Inspections', - origin: 'Ofsted', - use: 'Latest inspection outcomes for state-funded schools (as at April 2025). Averaged per postcode to give a local school quality score (1=Outstanding to 4=Inadequate).', - url: 'https://www.gov.uk/government/statistical-data-sets/monthly-management-information-ofsteds-school-inspections-outcomes', - license: 'Open Government Licence v3.0', - }, - { - id: 'broadband', - name: 'Ofcom Broadband Performance', - origin: 'Ofcom', - use: 'Fixed broadband coverage and speeds by Output Area from Connected Nations 2025. Includes max download/upload speeds across different speed tiers.', - url: 'https://www.ofcom.org.uk/phones-and-broadband/coverage-and-speeds/connected-nations-20252/data-downloads-2025', - license: 'Open Government Licence v3.0', - }, - { - id: 'geosure', - name: 'GeoSure Ground Stability', - origin: 'Ordnance Survey', - use: 'Ground stability hazard ratings on a 5km hex grid covering Great Britain. Six risk categories (collapsible deposits, compressible ground, landslides, running sand, shrink-swell, and soluble rocks) rated Low, Moderate, or Significant. Spatial-joined to postcodes via centroid intersection.', - url: 'https://osdatahub.os.uk/downloads/open/GeoSure', - license: 'Open Government Licence v3.0', - }, - { - id: 'council-tax', - name: 'Council Tax Levels 2025-26', - origin: 'Ministry of Housing, Communities & Local Government', - use: 'Annual council tax rates for Bands A-H for all 296 billing authorities in England, for a dwelling occupied by two adults. Joined to properties via local authority district code from the NSPL postcode lookup.', - url: 'https://www.gov.uk/government/statistics/council-tax-levels-set-by-local-authorities-in-england-2025-to-2026', - license: 'Open Government Licence v3.0', - }, -]; - -export default function DataSourcesPage() { - const [highlightedId, setHighlightedId] = useState(null); - const cardRefs = useRef>({}); - - useEffect(() => { - function handleHash() { - const hash = window.location.hash.replace('#', ''); - if (hash && DATA_SOURCES.some((s) => s.id === hash)) { - setHighlightedId(hash); - // Scroll after a brief delay to allow render - setTimeout(() => { - cardRefs.current[hash]?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - }, 100); - } else { - setHighlightedId(null); - } - } - handleHash(); - window.addEventListener('hashchange', handleHash); - return () => window.removeEventListener('hashchange', handleHash); - }, []); - - return ( -
-
-
-

Data Sources

-

- This application combines {DATA_SOURCES.length} open datasets covering property prices, - energy performance, transport, demographics, crime, environment, and more. -

-
- {DATA_SOURCES.map((source) => ( -
{ - cardRefs.current[source.id] = el; - }} - className={`bg-white dark:bg-navy-800 rounded-lg border p-5 ${ - highlightedId === source.id - ? 'border-teal-400 ring-2 ring-teal-400' - : 'border-warm-200 dark:border-navy-700' - }`} - > -
-

- {source.name} -

- - {source.license} - -
-

- Source: {source.origin} -

-

{source.use}

- - {source.url} - - {'optOutUrl' in source && source.optOutUrl && ( - - )} -
- ))} -
-
-
- - -
- ); -} diff --git a/frontend/src/components/learn/LearnPage.tsx b/frontend/src/components/learn/LearnPage.tsx index c04ccd4..c748804 100644 --- a/frontend/src/components/learn/LearnPage.tsx +++ b/frontend/src/components/learn/LearnPage.tsx @@ -117,6 +117,14 @@ const DATA_SOURCES = [ url: 'https://www.gov.uk/government/statistics/council-tax-levels-set-by-local-authorities-in-england-2025-to-2026', license: 'Open Government Licence v3.0', }, + { + id: 'ons-rental', + name: 'Private Rental Market Statistics', + origin: 'ONS / Valuation Office Agency', + use: 'Median monthly private rental prices by local authority and bedroom category (Oct 2022 - Sep 2023). Joined to properties via local authority district code and estimated bedroom count.', + url: 'https://www.ons.gov.uk/peoplepopulationandcommunity/housing/datasets/privaterentalmarketsummarystatisticsinengland', + license: 'Open Government Licence v3.0', + }, ]; interface FAQItem { diff --git a/frontend/src/components/map/AiFilterInput.tsx b/frontend/src/components/map/AiFilterInput.tsx new file mode 100644 index 0000000..1472a11 --- /dev/null +++ b/frontend/src/components/map/AiFilterInput.tsx @@ -0,0 +1,53 @@ +import { memo, useState, useCallback } from 'react'; +import { SpinnerIcon } from '../ui/icons/SpinnerIcon'; + +interface AiFilterInputProps { + loading: boolean; + error: string | null; + onSubmit: (query: string) => void; +} + +export default memo(function AiFilterInput({ loading, error, onSubmit }: AiFilterInputProps) { + const [query, setQuery] = useState(''); + + const handleSubmit = useCallback( + (e: React.FormEvent) => { + e.preventDefault(); + const trimmed = query.trim(); + if (!trimmed || loading) return; + onSubmit(trimmed); + }, + [query, loading, onSubmit] + ); + + return ( +
+
+ setQuery(e.target.value)} + placeholder="Describe your ideal property..." + className="flex-1 min-w-0 px-2.5 py-1.5 text-sm rounded-lg border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-800 text-warm-700 dark:text-warm-200 placeholder-warm-400 dark:placeholder-warm-500 focus:outline-none focus:ring-2 focus:ring-teal-400" + disabled={loading} + /> + +
+ {error && ( +

+ {error} +

+ )} +
+ ); +}); diff --git a/frontend/src/components/map/Filters.tsx b/frontend/src/components/map/Filters.tsx index 3e7a1ef..6e543af 100644 --- a/frontend/src/components/map/Filters.tsx +++ b/frontend/src/components/map/Filters.tsx @@ -1,7 +1,7 @@ -import { memo, useState, useMemo } from 'react'; +import { memo, useState, useMemo, useRef, useCallback } from 'react'; import { Slider } from '../ui/Slider'; -import { FilterIcon, LightbulbIcon } from '../ui/icons'; -import { EmptyState } from '../ui/EmptyState'; +import { LightbulbIcon } from '../ui/icons'; + import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader'; import { PillToggle } from '../ui/PillToggle'; import { PillGroup } from '../ui/PillGroup'; @@ -14,6 +14,7 @@ import InfoPopup from '../ui/InfoPopup'; import { FeatureInfoPopup } from '../ui/FeatureInfoPopup'; import { FeatureActions } from '../ui/FeatureIcons'; import { FeatureLabel } from '../ui/FeatureLabel'; +import AiFilterInput from './AiFilterInput'; import FeatureBrowser from './FeatureBrowser'; import { TravelTimeCard } from './TravelTimeCard'; import type { TransportMode } from '../../hooks/useTravelTime'; @@ -80,6 +81,9 @@ interface FiltersProps { onTravelTimeSetDestination: (lat: number, lon: number, label: string) => void; onTravelTimeModeChange: (mode: TransportMode) => void; onTravelTimeRangeChange: (range: [number, number]) => void; + aiFilterLoading: boolean; + aiFilterError: string | null; + onAiFilterSubmit: (query: string) => void; } export default memo(function Filters({ @@ -111,13 +115,27 @@ export default memo(function Filters({ onTravelTimeSetDestination, onTravelTimeModeChange, onTravelTimeRangeChange, + aiFilterLoading, + aiFilterError, + onAiFilterSubmit, }: FiltersProps) { const availableFeatures = features.filter((f) => !enabledFeatures.has(f.name)); const enabledFeatureList = features.filter((f) => enabledFeatures.has(f.name)); + const containerRef = useRef(null); const [showPhilosophy, setShowPhilosophy] = useState(false); const [activeInfoFeature, setActiveInfoFeature] = useState(null); const [collapsedGroups, toggleGroup] = useCollapsibleGroups(); + + const handleAddAndScroll = useCallback( + (name: string) => { + onAddFilter(name); + requestAnimationFrame(() => { + containerRef.current?.scrollTo({ top: 0, behavior: 'smooth' }); + }); + }, + [onAddFilter] + ); const enabledGroups = useMemo( () => groupFeaturesByCategory(enabledFeatureList), [enabledFeatureList] @@ -134,15 +152,18 @@ export default memo(function Filters({ }, [features]); return ( -
-
- +
+
+ +
+ +
@@ -176,12 +197,9 @@ export default memo(function Filters({ )} {enabledFeatureList.length === 0 && !travelTimeEnabled && ( - } - title="No active filters" - description="Browse features below and click + to add a filter" - className="px-3 py-4" - /> +

+ Browse features below and click + to add a filter +

)} {enabledGroups.map((group) => { @@ -304,7 +322,7 @@ export default memo(function Filters({ availableFeatures={availableFeatures} allFeatures={features} pinnedFeature={pinnedFeature} - onAddFilter={onAddFilter} + onAddFilter={handleAddAndScroll} onTogglePin={onTogglePin} onNavigateToSource={onNavigateToSource} openInfoFeature={openInfoFeature} diff --git a/frontend/src/components/map/MapPage.tsx b/frontend/src/components/map/MapPage.tsx index 99c0e87..35421e0 100644 --- a/frontend/src/components/map/MapPage.tsx +++ b/frontend/src/components/map/MapPage.tsx @@ -15,26 +15,26 @@ import { usePOIData } from '../../hooks/usePOIData'; import { useFilters } from '../../hooks/useFilters'; import { useHexagonSelection } from '../../hooks/useHexagonSelection'; import { usePaneResize } from '../../hooks/usePaneResize'; +import { useAiFilters } from '../../hooks/useAiFilters'; import { useAreaSummary } from '../../hooks/useAreaSummary'; import { useUrlSync } from '../../hooks/useUrlSync'; import { useTravelTime, type TravelTimeInitial } from '../../hooks/useTravelTime'; import { apiUrl, buildFilterString } from '../../lib/api'; import { SpinnerIcon } from '../ui/icons/SpinnerIcon'; +import { MapPinIcon } from '../ui/icons/MapPinIcon'; export interface ExportState { onExport: () => void; exporting: boolean; } -type MobileBottomTab = 'filters' | 'pois' | 'area'; - interface MapPageProps { features: FeatureMeta[]; poiCategoryGroups: POICategoryGroup[]; initialFilters: FeatureFilters; initialViewState: ViewState; initialPOICategories: Set; - initialTab: 'pois' | 'properties' | 'area'; + initialTab: 'properties' | 'area'; initialLoading: boolean; theme: 'light' | 'dark'; pendingInfoFeature: string | null; @@ -73,9 +73,11 @@ export default function MapPage({ const [rightPaneWidth, rightPaneHandlers] = usePaneResize(288, 200, 500, 'right'); // Mobile state - const [mobileBottomTab, setMobileBottomTab] = useState('filters'); const [mobileDrawerOpen, setMobileDrawerOpen] = useState(false); + // POI floating panel state + const [poiPaneOpen, setPoiPaneOpen] = useState(false); + // Initialize filters first const { filters, @@ -90,6 +92,7 @@ export default function MapPage({ handleAddFilter, handleFilterChange, handleRemoveFilter, + handleSetFilters, handleDragStart, handleDragChange, handleDragEnd, @@ -101,6 +104,16 @@ export default function MapPage({ features, }); + // AI filters hook + const aiFilters = useAiFilters(); + const handleAiFilterSubmit = useCallback( + async (query: string) => { + const result = await aiFilters.fetchAiFilters(query); + if (result) handleSetFilters(result); + }, + [aiFilters.fetchAiFilters, handleSetFilters] + ); + // Travel time hook const travelTime = useTravelTime(initialTravelTime); @@ -161,7 +174,6 @@ export default function MapPage({ handleHexagonClick(id, isPostcode); if (id) { setMobileDrawerOpen(true); - setMobileBottomTab('area'); } }, [handleHexagonClick] @@ -289,6 +301,10 @@ export default function MapPage({ screenshotMode ogMode={ogMode} bounds={mapData.bounds} + travelTimeEnabled={travelTime.enabled} + travelTimeDestination={travelTime.destination} + travelTimeColorRange={mapData.travelTimeColorRange} + travelTimeRange={travelTime.timeRange} />
); @@ -368,6 +384,9 @@ export default function MapPage({ onTravelTimeSetDestination={travelTime.handleSetDestination} onTravelTimeModeChange={travelTime.handleModeChange} onTravelTimeRangeChange={travelTime.handleTimeRangeChange} + aiFilterLoading={aiFilters.loading} + aiFilterError={aiFilters.error} + onAiFilterSubmit={handleAiFilterSubmit} /> ); @@ -421,6 +440,19 @@ export default function MapPage({ Loading...
)} + {/* Floating POI button */} + + {/* Floating POI panel */} + {poiPaneOpen && ( +
+ {renderPOIPane()} +
+ )}
{/* Bottom panel — 55% */} @@ -466,27 +498,9 @@ export default function MapPage({ inline /> )} - {/* Tab bar */} -
- setMobileBottomTab('filters')} - /> - setMobileBottomTab('pois')} - /> -
- - {/* Tab content */} + {/* Filters content */}
- {mobileBottomTab === 'pois' ? ( -
{renderPOIPane()}
- ) : ( - renderFilters() - )} + {renderFilters()}
@@ -496,7 +510,6 @@ export default function MapPage({ onClose={() => setMobileDrawerOpen(false)} renderArea={renderAreaPane} renderProperties={renderPropertiesPane} - renderPOIs={renderPOIPane} /> )}
@@ -565,6 +578,19 @@ export default function MapPage({ Loading... )} + {/* Floating POI button */} + + {/* Floating POI panel */} + {poiPaneOpen && ( +
+ {renderPOIPane()} +
+ )} {/* Right Pane */} @@ -590,19 +616,12 @@ export default function MapPage({ isActive={selection.rightPaneTab === 'properties'} onClick={selection.handlePropertiesTabClick} /> - selection.setRightPaneTab('pois')} - />
- {selection.rightPaneTab === 'area' - ? renderAreaPane() - : selection.rightPaneTab === 'properties' - ? renderPropertiesPane() - : renderPOIPane()} + {selection.rightPaneTab === 'properties' + ? renderPropertiesPane() + : renderAreaPane()}
diff --git a/frontend/src/components/map/MobileDrawer.tsx b/frontend/src/components/map/MobileDrawer.tsx index 20ec451..680b67d 100644 --- a/frontend/src/components/map/MobileDrawer.tsx +++ b/frontend/src/components/map/MobileDrawer.tsx @@ -2,20 +2,18 @@ import { useState, useEffect } from 'react'; import { CloseIcon } from '../ui/icons/CloseIcon'; import { TabButton } from '../ui/TabButton'; -type DrawerTab = 'area' | 'properties' | 'pois'; +type DrawerTab = 'area' | 'properties'; interface MobileDrawerProps { onClose: () => void; renderArea: () => React.ReactNode; renderProperties: () => React.ReactNode; - renderPOIs: () => React.ReactNode; } export default function MobileDrawer({ onClose, renderArea, renderProperties, - renderPOIs, }: MobileDrawerProps) { const [tab, setTab] = useState('area'); @@ -43,7 +41,6 @@ export default function MobileDrawer({ isActive={tab === 'properties'} onClick={() => setTab('properties')} /> - setTab('pois')} /> + + {showInfo && ( @@ -118,34 +137,6 @@ export default function POIPane({ onChange={setSearchTerm} placeholder="Search categories..." /> - -
-
- - -
- - {selectedCount}/{allCategories.length} selected - -
- - {selectedCount > 0 && ( -
- - {poiCount.toLocaleString()} POI{poiCount !== 1 ? 's' : ''} visible - -
- )}
diff --git a/frontend/src/components/map/PostcodeSearch.tsx b/frontend/src/components/map/PostcodeSearch.tsx index 20a1426..83c1b1b 100644 --- a/frontend/src/components/map/PostcodeSearch.tsx +++ b/frontend/src/components/map/PostcodeSearch.tsx @@ -1,14 +1,36 @@ import { useState, useCallback, useRef, useEffect } from 'react'; -import type { PostcodeGeometry } from '../../types'; -import { authHeaders } from '../../lib/api'; +import type { PostcodeGeometry, PlaceResult } from '../../types'; +import { authHeaders, logNonAbortError } from '../../lib/api'; import { useIsMobile } from '../../hooks/useIsMobile'; import { SearchIcon } from '../ui/icons/SearchIcon'; +import { MapPinIcon } from '../ui/icons/MapPinIcon'; export interface SearchedPostcode { postcode: string; geometry: PostcodeGeometry; } +const POSTCODE_RE = /^[A-Z]{1,2}\d[A-Z\d]?\s*\d?[A-Z]{0,2}$/i; +function looksLikePostcode(s: string) { + return POSTCODE_RE.test(s.trim()); +} + +type SearchResult = + | { type: 'postcode'; label: string } + | { type: 'place'; name: string; place_type: string; lat: number; lon: number }; + +const ZOOM_FOR_TYPE: Record = { + city: 10, + borough: 12, + town: 13, + suburb: 14, + neighbourhood: 14, + village: 14, + locality: 14, + hamlet: 15, + isolated_dwelling: 16, +}; + export default function PostcodeSearch({ onFlyTo, onPostcodeSearched, @@ -17,24 +39,29 @@ export default function PostcodeSearch({ onPostcodeSearched?: (postcode: SearchedPostcode | null) => void; }) { const [query, setQuery] = useState(''); + const [results, setResults] = useState([]); + const [activeIndex, setActiveIndex] = useState(-1); + const [open, setOpen] = useState(false); const [error, setError] = useState(null); const [loading, setLoading] = useState(false); const [expanded, setExpanded] = useState(false); const isMobile = useIsMobile(); - const formRef = useRef(null); + const containerRef = useRef(null); const inputRef = useRef(null); + const abortRef = useRef(null); + const debounceRef = useRef>(); - // Close on outside click (mobile only) + // Close on outside click useEffect(() => { - if (!isMobile || !expanded) return; const handler = (e: MouseEvent) => { - if (formRef.current && !formRef.current.contains(e.target as Node)) { - setExpanded(false); + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setOpen(false); + if (isMobile) setExpanded(false); } }; document.addEventListener('mousedown', handler); return () => document.removeEventListener('mousedown', handler); - }, [isMobile, expanded]); + }, [isMobile]); // Focus input when expanding on mobile useEffect(() => { @@ -43,16 +70,16 @@ export default function PostcodeSearch({ } }, [isMobile, expanded]); - const handleSubmit = useCallback( - async (e: React.FormEvent) => { - e.preventDefault(); - const trimmed = query.trim(); - if (!trimmed) return; - + const selectPostcode = useCallback( + async (postcode: string) => { setError(null); setLoading(true); + setOpen(false); try { - const res = await fetch(`/api/postcode/${encodeURIComponent(trimmed)}`, authHeaders()); + const res = await fetch( + `/api/postcode/${encodeURIComponent(postcode.trim())}`, + authHeaders() + ); if (!res.ok) { setError('Postcode not found'); return; @@ -66,6 +93,7 @@ export default function PostcodeSearch({ onFlyTo(json.latitude, json.longitude, 16); onPostcodeSearched?.({ postcode: json.postcode, geometry: json.geometry }); setQuery(''); + setResults([]); if (isMobile) setExpanded(false); } catch { setError('Lookup failed'); @@ -73,9 +101,115 @@ export default function PostcodeSearch({ setLoading(false); } }, - [query, onFlyTo, onPostcodeSearched, isMobile] + [onFlyTo, onPostcodeSearched, isMobile] ); + const selectPlace = useCallback( + (place: { name: string; place_type: string; lat: number; lon: number }) => { + const zoom = ZOOM_FOR_TYPE[place.place_type] ?? 14; + onFlyTo(place.lat, place.lon, zoom); + setQuery(''); + setResults([]); + setOpen(false); + if (isMobile) setExpanded(false); + }, + [onFlyTo, isMobile] + ); + + const selectResult = useCallback( + (result: SearchResult) => { + if (result.type === 'postcode') { + selectPostcode(result.label); + } else { + selectPlace(result); + } + }, + [selectPostcode, selectPlace] + ); + + const handleInputChange = useCallback((value: string) => { + setQuery(value); + setError(null); + setActiveIndex(-1); + + // Cancel in-flight request + abortRef.current?.abort(); + if (debounceRef.current) clearTimeout(debounceRef.current); + + const trimmed = value.trim(); + if (!trimmed) { + setResults([]); + setOpen(false); + return; + } + + if (looksLikePostcode(trimmed)) { + setResults([{ type: 'postcode', label: trimmed.toUpperCase() }]); + setOpen(true); + return; + } + + if (trimmed.length < 2) { + setResults([]); + setOpen(false); + return; + } + + // Debounced place search + debounceRef.current = setTimeout(async () => { + const controller = new AbortController(); + abortRef.current = controller; + try { + const params = new URLSearchParams({ q: trimmed, limit: '7' }); + const res = await fetch( + `/api/places?${params}`, + authHeaders({ signal: controller.signal }) + ); + if (!res.ok) return; + const json: { places: PlaceResult[] } = await res.json(); + const placeResults: SearchResult[] = json.places.map((p) => ({ + type: 'place' as const, + ...p, + })); + setResults(placeResults); + setOpen(placeResults.length > 0); + } catch (err) { + logNonAbortError('places search', err); + } + }, 200); + }, []); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === 'ArrowDown') { + e.preventDefault(); + setActiveIndex((prev) => (prev < results.length - 1 ? prev + 1 : prev)); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + setActiveIndex((prev) => (prev > 0 ? prev - 1 : -1)); + } else if (e.key === 'Enter') { + e.preventDefault(); + if (activeIndex >= 0 && activeIndex < results.length) { + selectResult(results[activeIndex]); + } else if (looksLikePostcode(query)) { + selectPostcode(query); + } + } else if (e.key === 'Escape') { + setOpen(false); + inputRef.current?.blur(); + } + }, + [results, activeIndex, query, selectResult, selectPostcode] + ); + + // Cleanup on unmount + useEffect(() => { + return () => { + abortRef.current?.abort(); + if (debounceRef.current) clearTimeout(debounceRef.current); + }; + }, []); + // Mobile collapsed state: just a search icon button if (isMobile && !expanded) { return ( @@ -83,7 +217,7 @@ export default function PostcodeSearch({ type="button" onClick={() => setExpanded(true)} className="absolute top-3 left-3 z-10 p-2 bg-white dark:bg-warm-800 rounded shadow-lg" - aria-label="Search postcode" + aria-label="Search places or postcodes" > @@ -91,36 +225,76 @@ export default function PostcodeSearch({ } return ( -
-
- { - setQuery(e.target.value); - setError(null); - }} - placeholder="Search postcode..." - className="px-3 py-2 text-sm w-40 border-none outline-none bg-white dark:bg-navy-800 dark:text-white dark:placeholder-warm-400" - /> - +
+
+
+ + handleInputChange(e.target.value)} + onFocus={() => { + if (results.length > 0) setOpen(true); + }} + onKeyDown={handleKeyDown} + placeholder="Search places or postcodes..." + className="px-2 py-2 text-sm w-56 border-none outline-none bg-transparent text-warm-700 dark:text-warm-200 placeholder-warm-400 dark:placeholder-warm-500" + /> + {loading && ( +
+ )} +
+ + {open && results.length > 0 && ( +
+ {results.map((result, idx) => ( + + ))} +
+ )}
+ {error && ( - + {error} )} - +
); } diff --git a/frontend/src/components/map/PropertiesPane.tsx b/frontend/src/components/map/PropertiesPane.tsx index 6b898aa..e7bac8d 100644 --- a/frontend/src/components/map/PropertiesPane.tsx +++ b/frontend/src/components/map/PropertiesPane.tsx @@ -248,6 +248,23 @@ function PropertyCard({ property }: { property: Property }) {
) : null}
+ + {property.renovation_history && property.renovation_history.length > 0 && ( +
+
Renovations
+
+ {property.renovation_history.map((reno, idx) => ( + + {reno.event} + {reno.year} + + ))} +
+
+ )}
); } diff --git a/frontend/src/components/map/TravelTimeCard.tsx b/frontend/src/components/map/TravelTimeCard.tsx index e9cb57c..b702b4c 100644 --- a/frontend/src/components/map/TravelTimeCard.tsx +++ b/frontend/src/components/map/TravelTimeCard.tsx @@ -11,9 +11,10 @@ import { authHeaders } from '../../lib/api'; import type { TransportMode } from '../../hooks/useTravelTime'; const MODES: { value: TransportMode; label: string }[] = [ - { value: 'transit', label: 'Transit' }, { value: 'car', label: 'Car' }, { value: 'bicycle', label: 'Bicycle' }, + { value: 'walking', label: 'Walking' }, + { value: 'transit', label: 'Transit' }, ]; interface TravelTimeCardProps { diff --git a/frontend/src/hooks/useAiFilters.ts b/frontend/src/hooks/useAiFilters.ts new file mode 100644 index 0000000..c6ccdcd --- /dev/null +++ b/frontend/src/hooks/useAiFilters.ts @@ -0,0 +1,55 @@ +import { useState, useCallback, useRef } from 'react'; +import type { FeatureFilters } from '../types'; +import { apiUrl, authHeaders, logNonAbortError } from '../lib/api'; + +interface UseAiFiltersResult { + fetchAiFilters: (query: string) => Promise; + loading: boolean; + error: string | null; +} + +export function useAiFilters(): UseAiFiltersResult { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const abortRef = useRef(null); + + const fetchAiFilters = useCallback(async (query: string): Promise => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + + setLoading(true); + setError(null); + + try { + const url = apiUrl('ai-filters'); + const response = await fetch( + url, + authHeaders({ + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query }), + signal: controller.signal, + }) + ); + + if (!response.ok) { + const text = await response.text(); + throw new Error(text || `HTTP ${response.status}`); + } + + const json = await response.json(); + setLoading(false); + return json.filters as FeatureFilters; + } catch (err) { + if (controller.signal.aborted) return null; + logNonAbortError('ai-filters', err); + const message = err instanceof Error ? err.message : 'Failed to generate filters'; + setError(message); + setLoading(false); + return null; + } + }, []); + + return { fetchAiFilters, loading, error }; +} diff --git a/frontend/src/hooks/useDeckLayers.ts b/frontend/src/hooks/useDeckLayers.ts index 1bb5918..4248813 100644 --- a/frontend/src/hooks/useDeckLayers.ts +++ b/frontend/src/hooks/useDeckLayers.ts @@ -11,12 +11,8 @@ import type { Bounds, } from '../types'; import type { SearchedPostcode } from '../components/map/PostcodeSearch'; -import { - emojiToTwemojiUrl, - DENSITY_GRADIENT, - DENSITY_GRADIENT_DARK, - getFeatureFillColor, -} from '../lib/map-utils'; +import { DENSITY_GRADIENT, DENSITY_GRADIENT_DARK } from '../lib/consts'; +import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils'; /** Convert POI id (e.g. "n12345") to OpenStreetMap URL */ function osmIdToUrl(id: string): string | null { @@ -242,7 +238,7 @@ export function useDeckLayers({ }, []); // --- Color triggers --- - const ttTrigger = `${travelTimeEnabled}|${travelTimeColorRange?.[0]}|${travelTimeColorRange?.[1]}|${travelTimeRange?.[0]}|${travelTimeRange?.[1]}|${travelTimeDestination?.[0]}`; + const ttTrigger = `${travelTimeEnabled}|${travelTimeColorRange?.[0]}|${travelTimeColorRange?.[1]}|${travelTimeRange?.[0]}|${travelTimeRange?.[1]}|${travelTimeDestination?.[0]}|${travelTimeDestination?.[1]}`; const colorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${countRange.min}|${countRange.max}|${selectedHexagonId}|${hoveredHexagonId}|${theme}|${ttTrigger}`; const postcodeColorTrigger = `${viewFeature}|${colorRange?.[0]}|${colorRange?.[1]}|${filterRange?.[0]}|${filterRange?.[1]}|${postcodeCountRange.min}|${postcodeCountRange.max}|${selectedPostcode}|${hoveredPostcode}|${theme}|${ttTrigger}`; diff --git a/frontend/src/hooks/useFilters.ts b/frontend/src/hooks/useFilters.ts index 7aa8b18..25d2bd2 100644 --- a/frontend/src/hooks/useFilters.ts +++ b/frontend/src/hooks/useFilters.ts @@ -118,6 +118,14 @@ export function useFilters({ initialFilters, features }: UseFiltersOptions) { } }, [activeFeature, dragValue]); + const handleSetFilters = useCallback((newFilters: FeatureFilters) => { + setFilters(newFilters); + setActiveFeature(null); + setDragValue(null); + setDragData(null); + setPinnedFeature(null); + }, []); + const handleTogglePin = useCallback((name: string) => { setPinnedFeature((prev) => (prev === name ? null : name)); }, []); @@ -144,6 +152,7 @@ export function useFilters({ initialFilters, features }: UseFiltersOptions) { handleAddFilter, handleFilterChange, handleRemoveFilter, + handleSetFilters, handleDragStart, handleDragChange, handleDragEnd, diff --git a/frontend/src/hooks/useHexagonSelection.ts b/frontend/src/hooks/useHexagonSelection.ts index e48564f..098ead8 100644 --- a/frontend/src/hooks/useHexagonSelection.ts +++ b/frontend/src/hooks/useHexagonSelection.ts @@ -29,7 +29,7 @@ export function useHexagonSelection({ filters, features, resolution }: UseHexago const [areaStats, setAreaStats] = useState(null); const [loadingAreaStats, setLoadingAreaStats] = useState(false); const [hoveredHexagon, setHoveredHexagon] = useState(null); - const [rightPaneTab, setRightPaneTab] = useState<'pois' | 'properties' | 'area'>('pois'); + const [rightPaneTab, setRightPaneTab] = useState<'properties' | 'area'>('area'); const fetchHexagonStats = useCallback( async (h3: string, res: number, signal?: AbortSignal, fields?: string[]) => { diff --git a/frontend/src/hooks/useTravelTime.ts b/frontend/src/hooks/useTravelTime.ts index 1f8b245..4f9d60a 100644 --- a/frontend/src/hooks/useTravelTime.ts +++ b/frontend/src/hooks/useTravelTime.ts @@ -1,6 +1,6 @@ import { useState, useCallback } from 'react'; -export type TransportMode = 'transit' | 'car' | 'bicycle'; +export type TransportMode = 'car' | 'bicycle' | 'walking' | 'transit'; export interface TravelTimeState { enabled: boolean; @@ -23,7 +23,7 @@ export function useTravelTime(initial?: TravelTimeInitial) { initial?.destination ?? null ); const [destinationLabel, setDestinationLabel] = useState(initial?.destinationLabel ?? ''); - const [mode, setMode] = useState(initial?.mode ?? 'transit'); + const [mode, setMode] = useState(initial?.mode ?? 'car'); const [timeRange, setTimeRange] = useState<[number, number] | null>( initial?.timeRange ?? null ); diff --git a/frontend/src/hooks/useUrlSync.ts b/frontend/src/hooks/useUrlSync.ts index 6299721..b3dea39 100644 --- a/frontend/src/hooks/useUrlSync.ts +++ b/frontend/src/hooks/useUrlSync.ts @@ -18,7 +18,7 @@ export function useUrlSync( filters: FeatureFilters, features: FeatureMeta[], selectedPOICategories: Set, - rightPaneTab: 'pois' | 'properties' | 'area', + rightPaneTab: 'properties' | 'area', travelTime?: TravelTimeUrlState ) { const urlDebounceRef = useRef | null>(null); diff --git a/frontend/src/lib/map-utils.ts b/frontend/src/lib/map-utils.ts index 80f65c4..7abf29e 100644 --- a/frontend/src/lib/map-utils.ts +++ b/frontend/src/lib/map-utils.ts @@ -10,14 +10,6 @@ import { BUFFER_MULTIPLIER, } from './consts'; -// Re-export constants for backwards compatibility -export { - FEATURE_GRADIENT as GRADIENT, - DENSITY_GRADIENT, - DENSITY_GRADIENT_DARK, - POSTCODE_ZOOM_THRESHOLD, -} from './consts'; - const ROAD_OPACITY = 0.4; export function getMapStyle(theme: 'light' | 'dark'): StyleSpecification { diff --git a/frontend/src/lib/url-state.ts b/frontend/src/lib/url-state.ts index 7908d78..4049ecd 100644 --- a/frontend/src/lib/url-state.ts +++ b/frontend/src/lib/url-state.ts @@ -31,7 +31,7 @@ export function parseUrlState(): { viewState?: ViewState; filters?: FeatureFilters; poiCategories?: Set; - tab?: 'pois' | 'properties' | 'area'; + tab?: 'properties' | 'area'; travelTime?: TravelTimeInitial; } { const params = new URLSearchParams(window.location.search); @@ -61,7 +61,7 @@ export function parseUrlState(): { // Tab: full name const tab = params.get('tab'); - if (tab === 'properties' || tab === 'pois' || tab === 'area') { + if (tab === 'properties' || tab === 'area') { result.tab = tab; } @@ -94,7 +94,7 @@ export function stateToParams( filters: FeatureFilters, features: FeatureMeta[], selectedPOICategories: Set, - rightPaneTab: 'pois' | 'properties' | 'area', + rightPaneTab: 'properties' | 'area', travelTime?: { enabled: boolean; destination: [number, number] | null; destinationLabel: string; mode: string; timeRange: [number, number] | null } ): URLSearchParams { const params = new URLSearchParams(); @@ -121,8 +121,6 @@ export function stateToParams( if (rightPaneTab === 'properties') { params.set('tab', 'properties'); - } else if (rightPaneTab === 'area') { - params.set('tab', 'area'); } if (travelTime?.enabled && travelTime.destination) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 791fc29..6eec4ab 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -99,6 +99,18 @@ export interface POICategoriesResponse { groups: POICategoryGroup[]; } +export interface PlaceResult { + name: string; + place_type: string; + lat: number; + lon: number; +} + +export interface RenovationEvent { + year: number; + event: string; +} + export interface Property { // String fields address?: string; @@ -114,9 +126,10 @@ export interface Property { lon: number; is_construction_date_approximate?: boolean; + renovation_history?: RenovationEvent[]; // All other numeric features (dynamic, including construction_age_band) - [key: string]: string | number | boolean | undefined; + [key: string]: string | number | boolean | RenovationEvent[] | undefined; } export interface HexagonPropertiesResponse { diff --git a/pipeline/download/greenspace_water.py b/pipeline/download/greenspace_water.py index dde12ad..e8554d6 100644 --- a/pipeline/download/greenspace_water.py +++ b/pipeline/download/greenspace_water.py @@ -99,26 +99,26 @@ def main(): "--output", type=Path, required=True, help="Output parquet file path" ) parser.add_argument( - "--pbf", type=Path, default=None, help="Path to existing PBF file" + "--pbf", type=Path, required=True, help="Path to existing PBF file" ) args = parser.parse_args() - if args.pbf and args.pbf.exists(): + if args.pbf.exists(): pbf_file = args.pbf print(f"Using existing PBF: {pbf_file}") else: - pbf_file = Path("data/great-britain-latest.osm.pbf") - if not pbf_file.exists(): - download_pbf(pbf_file) - else: - print(f"Using cached PBF: {pbf_file}") + download_pbf(args.pbf) print("Extracting greenspace/water areas from PBF (two-pass area assembly)...") - with tqdm(unit=" areas", unit_scale=True, desc="Processing", smoothing=0.05) as progress: + with tqdm( + unit=" areas", unit_scale=True, desc="Processing", smoothing=0.05 + ) as progress: handler = GreenspaceHandler(progress) handler.apply_file(str(pbf_file), locations=True) - print(f"Found {len(handler.geometries)} greenspace/water polygons >= {MIN_AREA_SQM} sqm") + print( + f"Found {len(handler.geometries)} greenspace/water polygons >= {MIN_AREA_SQM} sqm" + ) # Merge overlapping geometries per 10km grid cell for efficiency if handler.geometries: diff --git a/pipeline/download/places.py b/pipeline/download/places.py new file mode 100644 index 0000000..68b51d7 --- /dev/null +++ b/pipeline/download/places.py @@ -0,0 +1,99 @@ +"""Extract place=* nodes from OSM PBF → data/places.parquet. + +Extracts named place nodes (cities, towns, suburbs, etc.) for typeahead search. +Reuses the same great-britain-latest.osm.pbf as pois.py. +""" + +import argparse +from pathlib import Path + +import osmium +import polars as pl +from tqdm import tqdm + +from .pois import UK_BBOX_EAST, UK_BBOX_NORTH, UK_BBOX_SOUTH, UK_BBOX_WEST, download_pbf + +PLACE_TYPES = { + "city", + "borough", + "town", + "suburb", + "neighbourhood", + "village", + "hamlet", + "locality", + "isolated_dwelling", +} + + +class PlaceHandler(osmium.SimpleHandler): + def __init__(self, progress: tqdm) -> None: + super().__init__() + self._progress = progress + self.places: list[dict] = [] + + def node(self, n: osmium.osm.Node) -> None: + self._progress.update(1) + if not n.location.valid: + return + lat, lon = n.location.lat, n.location.lon + if not (UK_BBOX_SOUTH <= lat <= UK_BBOX_NORTH and UK_BBOX_WEST <= lon <= UK_BBOX_EAST): + return + place_type = n.tags.get("place") + if place_type not in PLACE_TYPES: + return + name = n.tags.get("name:en", n.tags.get("name", "")) + if not name: + return + self.places.append( + {"name": name, "place_type": place_type, "lat": lat, "lon": lon} + ) + self._progress.set_postfix(places=f"{len(self.places):,}", refresh=False) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Extract place names from OSM PBF" + ) + parser.add_argument( + "--output", type=Path, required=True, help="Output parquet file path" + ) + parser.add_argument( + "--pbf", type=Path, default=None, help="Path to existing PBF file (skips download)" + ) + args = parser.parse_args() + + if args.pbf and args.pbf.exists(): + pbf_file = args.pbf + print(f"Using existing PBF: {pbf_file}") + else: + pbf_file = Path("data/great-britain-latest.osm.pbf") + if not pbf_file.exists(): + download_pbf(pbf_file) + else: + print(f"Using cached PBF: {pbf_file}") + + print(f"Extracting place nodes: {sorted(PLACE_TYPES)}") + with tqdm( + unit=" elements", + unit_scale=True, + desc="Streaming", + smoothing=0.05, + mininterval=1.0, + ) as progress: + handler = PlaceHandler(progress) + handler.apply_file(str(pbf_file), locations=True) + + print(f"Extracted {len(handler.places):,} place nodes") + + if handler.places: + df = pl.DataFrame(handler.places) + args.output.parent.mkdir(parents=True, exist_ok=True) + df.write_parquet(args.output) + print(f"Saved to {args.output}") + else: + print("No places found — skipping output") + + +if __name__ == "__main__": + main() diff --git a/pipeline/download/rental_prices.py b/pipeline/download/rental_prices.py new file mode 100644 index 0000000..d2787a3 --- /dev/null +++ b/pipeline/download/rental_prices.py @@ -0,0 +1,89 @@ +import argparse +import tempfile + +import polars as pl +from pathlib import Path + +from pipeline.utils import download + +URL = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/housing/datasets/privaterentalmarketsummarystatisticsinengland/october2022toseptember2023/privaterentalmarketstatistics231220.xls" + +# Sheets 12-16 are LA-level breakdowns: Studio, 1 Bed, 2 Bed, 3 Bed, 4+ Bed +# (Sheet 11 is "Room" — shared house rooms, not self-contained, so skip it) +BEDROOM_SHEETS = { + 12: 0, # Studio + 13: 1, # One Bedroom + 14: 2, # Two Bedrooms + 15: 3, # Three Bedrooms + 16: 4, # Four or more Bedrooms +} + +# Local authority district codes in England +LA_PREFIXES = ("E06", "E07", "E08", "E09") + + +def _read_sheet(xls_path: Path, sheet_id: int, bedrooms: int) -> pl.DataFrame: + """Read one bedroom category sheet, extract LA-level median rents.""" + df = pl.read_excel(xls_path, sheet_id=sheet_id) + + # Columns are unnamed; positional: + # 0=LA Code, 1=Area Code, 2=Area Name, 3=Count, 4=Mean, 5=LQ, 6=Median, 7=UQ + # First 4 rows are headers (title, notes, bedroom label, column headers) + df = df.slice(4) + + area_code_col = df.columns[1] + median_col = df.columns[6] + + return ( + df.select( + pl.col(area_code_col).alias("area_code"), + pl.col(median_col).alias("median_monthly_rent"), + ) + .filter( + pl.col("area_code").is_not_null() + & pl.col("area_code").str.starts_with("E06") + | pl.col("area_code").str.starts_with("E07") + | pl.col("area_code").str.starts_with("E08") + | pl.col("area_code").str.starts_with("E09") + ) + .with_columns( + # Suppressed values are ".." — cast will turn them to null + pl.col("median_monthly_rent").cast(pl.Float32, strict=False), + pl.lit(bedrooms).cast(pl.UInt8).alias("bedrooms"), + ) + ) + + +def convert_to_parquet(xls_path: Path, parquet_path: Path) -> None: + frames = [] + for sheet_id, bedrooms in BEDROOM_SHEETS.items(): + df = _read_sheet(xls_path, sheet_id, bedrooms) + print(f" Sheet {sheet_id} (bedrooms={bedrooms}): {df.height} rows") + frames.append(df) + + combined = pl.concat(frames) + print(f"Combined: {combined.shape}") + print(f"Non-null medians: {combined['median_monthly_rent'].drop_nulls().len()}") + print(combined.head(10)) + + combined.write_parquet(parquet_path, compression="zstd") + print(f"Saved to {parquet_path}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Download and convert ONS private rental market statistics" + ) + parser.add_argument( + "--output", type=Path, required=True, help="Output parquet file path" + ) + args = parser.parse_args() + + with tempfile.TemporaryDirectory() as cache_dir: + xls_path = Path(cache_dir) / "rental_prices.xls" + download(URL, xls_path, timeout=60) + convert_to_parquet(xls_path, args.output) + + +if __name__ == "__main__": + main() diff --git a/pipeline/transform/join_epc_pp.py b/pipeline/transform/join_epc_pp.py index 789fcac..d2ab075 100644 --- a/pipeline/transform/join_epc_pp.py +++ b/pipeline/transform/join_epc_pp.py @@ -6,6 +6,8 @@ from ..utils import fuzzy_join_on_postcode pl.Config.set_tbl_cols(-1) +RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7} + def main(): parser = argparse.ArgumentParser(description="Fuzzy join EPC and Price Paid data") @@ -20,7 +22,7 @@ def main(): ) args = parser.parse_args() - epc = ( + epc_base = ( pl.scan_csv(args.epc) .select( pl.col("ADDRESS").alias("epc_address"), @@ -42,11 +44,90 @@ def main(): .otherwise(pl.col("NUMBER_HABITABLE_ROOMS")) .alias("NUMBER_HABITABLE_ROOMS"), ) - .sort("INSPECTION_DATE", descending=True) + ) + + # Dedup fork: keep latest certificate per property (existing logic) + epc = ( + epc_base.sort("INSPECTION_DATE", descending=True) .group_by("epc_address", "POSTCODE") .first() ) + # Events fork: detect renovation events between consecutive certificates + # Collect eagerly because .over() window functions don't work in streaming + # engine (fuzzy_join.py:50 uses sink_parquet which requires streaming). + events = ( + epc_base.sort("INSPECTION_DATE") + .with_columns( + pl.col("CURRENT_ENERGY_RATING") + .replace_strict(RATING_RANK, default=None, return_dtype=pl.Int32) + .alias("_rating_rank"), + ) + .with_columns( + pl.col("NUMBER_HABITABLE_ROOMS") + .shift(1) + .over("epc_address", "POSTCODE") + .alias("_prev_rooms"), + pl.col("TOTAL_FLOOR_AREA") + .shift(1) + .over("epc_address", "POSTCODE") + .alias("_prev_area"), + pl.col("_rating_rank") + .shift(1) + .over("epc_address", "POSTCODE") + .alias("_prev_rating_rank"), + ) + .with_columns( + pl.when( + pl.col("NUMBER_HABITABLE_ROOMS").is_not_null() + & pl.col("_prev_rooms").is_not_null() + & (pl.col("NUMBER_HABITABLE_ROOMS") != pl.col("_prev_rooms")) + ) + .then(pl.lit("Remodeling")) + .when( + pl.col("TOTAL_FLOOR_AREA").is_not_null() + & pl.col("_prev_area").is_not_null() + & (pl.col("TOTAL_FLOOR_AREA") > pl.col("_prev_area")) + ) + .then(pl.lit("Extension")) + .when( + pl.col("_rating_rank").is_not_null() + & pl.col("_prev_rating_rank").is_not_null() + & (pl.col("_rating_rank") < pl.col("_prev_rating_rank")) + ) + .then(pl.lit("Renovation")) + .otherwise(pl.lit(None, dtype=pl.String)) + .alias("_event"), + ) + .filter(pl.col("_event").is_not_null()) + .with_columns( + pl.col("INSPECTION_DATE") + .cast(pl.String) + .str.slice(0, 4) + .cast(pl.Int32) + .alias("_event_year"), + ) + .group_by("epc_address", "POSTCODE") + .agg( + pl.struct( + pl.col("_event_year").alias("year"), + pl.col("_event").alias("event"), + ).alias("renovation_history"), + ) + .collect() + ) + + event_counts = events["renovation_history"].explode().struct.field("event").value_counts() + print(f"Renovation events: {events.height} properties with events") + print(event_counts) + + # Left-join events back onto dedup EPC + epc = epc.join( + events.lazy(), + on=["epc_address", "POSTCODE"], + how="left", + ) + print("EPC dataset") print(epc.head().collect()) diff --git a/pipeline/transform/merge.py b/pipeline/transform/merge.py index 707f394..fd0fda4 100644 --- a/pipeline/transform/merge.py +++ b/pipeline/transform/merge.py @@ -42,6 +42,7 @@ def _build_wide( school_proximity_path: Path, broadband_path: Path, geosure_path: Path, + rental_prices_path: Path, ) -> pl.DataFrame: """Build the wide dataframe by joining epc_pp with all auxiliary data.""" wide = ( @@ -94,6 +95,21 @@ def _build_wide( how="left", ) + # Derive bedroom count: habitable rooms - 1 (assuming 1 reception room), clipped to 0..4 + wide = wide.with_columns( + (pl.col("number_habitable_rooms") - 1) + .clip(0, 4) + .cast(pl.UInt8) + .alias("_bedrooms"), + ) + rental = pl.scan_parquet(rental_prices_path) + wide = wide.join( + rental, + left_on=["Local Authority District code (2024)", "_bedrooms"], + right_on=["area_code", "bedrooms"], + how="left", + ) + crime = pl.scan_parquet(crime_path) wide = wide.join(crime, left_on="lsoa21", right_on="LSOA code", how="left") @@ -208,6 +224,7 @@ def _build_wide( .drop( "inspection_date", "floor_height", + "_bedrooms", "LSOA name (2021)", "Local Authority District code (2024)", "Local Authority District name (2024)", @@ -258,6 +275,7 @@ def _build_wide( "running_sand_risk": "Running sand risk", "shrink_swell_risk": "Shrink-swell risk", "soluble_rocks_risk": "Soluble rocks risk", + "median_monthly_rent": "Estimated monthly rent", } ) ) @@ -332,6 +350,12 @@ def main(): required=True, help="GeoSure ground stability parquet file", ) + parser.add_argument( + "--rental-prices", + type=Path, + required=True, + help="ONS rental prices by LA and bedroom count parquet file", + ) parser.add_argument( "--output", type=Path, required=True, help="Output parquet file path" ) @@ -350,6 +374,7 @@ def main(): school_proximity_path=args.school_proximity, broadband_path=args.broadband, geosure_path=args.geosure, + rental_prices_path=args.rental_prices, ) print(f"Columns: {wide.columns}") diff --git a/r5-java/Dockerfile b/r5-java/Dockerfile index 73a6e41..e0b5667 100644 --- a/r5-java/Dockerfile +++ b/r5-java/Dockerfile @@ -1,26 +1,11 @@ FROM eclipse-temurin:21-jdk AS build WORKDIR /app -# Download pre-built R5 fat JAR from GitHub Releases +# Download pre-built R5 fat JAR from GitHub Releases (includes all R5 deps) ADD https://github.com/conveyal/r5/releases/download/v7.5/r5-v7.5-all.jar /app/lib/r5.jar -# Download Javalin + Gson + SLF4J -ADD https://repo1.maven.org/maven2/io/javalin/javalin/6.4.0/javalin-6.4.0.jar /app/lib/javalin.jar +# Gson for JSON (HTTP server is built into JDK) ADD https://repo1.maven.org/maven2/com/google/code/gson/gson/2.11.0/gson-2.11.0.jar /app/lib/gson.jar -ADD https://repo1.maven.org/maven2/org/slf4j/slf4j-simple/2.0.16/slf4j-simple-2.0.16.jar /app/lib/slf4j-simple.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-server/11.0.24/jetty-server-11.0.24.jar /app/lib/jetty-server.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-util/11.0.24/jetty-util-11.0.24.jar /app/lib/jetty-util.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-http/11.0.24/jetty-http-11.0.24.jar /app/lib/jetty-http.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-io/11.0.24/jetty-io-11.0.24.jar /app/lib/jetty-io.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-servlet/11.0.24/jetty-servlet-11.0.24.jar /app/lib/jetty-servlet.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-security/11.0.24/jetty-security-11.0.24.jar /app/lib/jetty-security.jar -ADD https://repo1.maven.org/maven2/jakarta/servlet/jakarta.servlet-api/5.0.0/jakarta.servlet-api-5.0.0.jar /app/lib/servlet-api.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/websocket/websocket-jetty-server/11.0.24/websocket-jetty-server-11.0.24.jar /app/lib/ws-server.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/websocket/websocket-jetty-api/11.0.24/websocket-jetty-api-11.0.24.jar /app/lib/ws-api.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/websocket/websocket-core-server/11.0.24/websocket-core-server-11.0.24.jar /app/lib/ws-core-server.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/websocket/websocket-core-common/11.0.24/websocket-core-common-11.0.24.jar /app/lib/ws-core-common.jar -ADD https://repo1.maven.org/maven2/org/eclipse/jetty/websocket/websocket-servlet/11.0.24/websocket-servlet-11.0.24.jar /app/lib/ws-servlet.jar -ADD https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib/2.1.0/kotlin-stdlib-2.1.0.jar /app/lib/kotlin-stdlib.jar COPY src/ src/ RUN javac -cp "lib/*" -d out src/main/java/propertymap/App.java @@ -30,4 +15,6 @@ WORKDIR /app RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* COPY --from=build /app/lib/ /app/lib/ COPY --from=build /app/out/ /app/out/ -ENTRYPOINT ["java", "-Xmx4g", "-cp", "out:lib/*", "propertymap.App"] +COPY entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/r5-java/build.gradle b/r5-java/build.gradle deleted file mode 100644 index b23e558..0000000 --- a/r5-java/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -plugins { - id 'java' - id 'application' - id 'com.github.johnrengelman.shadow' version '8.1.1' -} - -repositories { - mavenCentral() -} - -java { - toolchain { - languageVersion = JavaLanguageVersion.of(21) - } -} - -application { - mainClass = 'propertymap.App' -} - -dependencies { - implementation 'com.conveyal:r5:7.2' - implementation 'io.javalin:javalin:6.4.0' - implementation 'com.google.code.gson:gson:2.11.0' - implementation 'org.slf4j:slf4j-simple:2.0.16' -} - -jar { - manifest { - attributes 'Main-Class': 'propertymap.App' - } -} - -shadowJar { - archiveClassifier = '' - mergeServiceFiles() -} diff --git a/r5-java/entrypoint.sh b/r5-java/entrypoint.sh new file mode 100644 index 0000000..22ff069 --- /dev/null +++ b/r5-java/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +TRANSIT_DIR=$DATA_DIR +NETWORK_DIR=$NETWORK_CACHE_DIR +BUILD_DIR="$NETWORK_DIR/build" + +# If no cached network yet, copy transit data to a writable location for the build. +# R5 writes temp files (.mapdb) next to the OSM/GTFS files during network construction. +if [ ! -f "$NETWORK_DIR/network.dat" ]; then + echo "No cached network — copying transit data to writable build dir..." + mkdir -p "$BUILD_DIR" + cp "$OSM_DIR"/*.osm.pbf "$BUILD_DIR/" 2>/dev/null || true + cp "$TRANSIT_DIR"/*.zip "$BUILD_DIR/" 2>/dev/null || true + export DATA_DIR="$BUILD_DIR" +fi + +exec java -Xmx16g -cp "out:lib/*" propertymap.App diff --git a/r5-java/settings.gradle b/r5-java/settings.gradle deleted file mode 100644 index 7677d00..0000000 --- a/r5-java/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = 'r5-service' diff --git a/r5-java/src/main/java/propertymap/App.java b/r5-java/src/main/java/propertymap/App.java index ce57d0a..63bb990 100644 --- a/r5-java/src/main/java/propertymap/App.java +++ b/r5-java/src/main/java/propertymap/App.java @@ -1,16 +1,26 @@ package propertymap; +import com.conveyal.r5.OneOriginResult; import com.conveyal.r5.analyst.FreeFormPointSet; +import com.conveyal.r5.analyst.PointSet; import com.conveyal.r5.analyst.TravelTimeComputer; +import com.conveyal.r5.analyst.WebMercatorExtents; import com.conveyal.r5.analyst.cluster.RegionalTask; +import com.conveyal.r5.analyst.cluster.TravelTimeResult; import com.conveyal.r5.api.util.LegMode; import com.conveyal.r5.api.util.TransitModes; +import com.conveyal.r5.kryo.KryoNetworkSerializer; import com.conveyal.r5.transit.TransportNetwork; import com.google.gson.Gson; -import io.javalin.Javalin; -import io.javalin.http.Context; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.locationtech.jts.geom.Coordinate; import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.util.EnumSet; @@ -30,10 +40,17 @@ public class App { public static void main(String[] args) throws Exception { String dataDir = System.getenv("DATA_DIR"); - if (dataDir == null) dataDir = "/data/transit"; + + if (dataDir == null) { + System.err.println("Error: DATA_DIR environment variable not set"); + System.exit(1); + } String networkCacheDir = System.getenv("NETWORK_CACHE_DIR"); - if (networkCacheDir == null) networkCacheDir = "/data/network"; + if (networkCacheDir == null) { + System.err.println("Error: NETWORK_CACHE_DIR environment variable not set"); + System.exit(1); + } System.out.println("Loading transport network from " + dataDir); System.out.println("Network cache dir: " + networkCacheDir); @@ -41,51 +58,80 @@ public class App { File cacheFile = new File(networkCacheDir, "network.dat"); if (cacheFile.exists()) { System.out.println("Loading cached network from " + cacheFile); - network = TransportNetwork.read(cacheFile); + network = KryoNetworkSerializer.read(cacheFile); } else { System.out.println("Building network (first run, this takes a few minutes)..."); network = TransportNetwork.fromDirectory(new File(dataDir)); new File(networkCacheDir).mkdirs(); - network.write(cacheFile); + KryoNetworkSerializer.write(network, cacheFile); System.out.println("Network cached to " + cacheFile); } + // Build stop-to-vertex distance tables (needed for egress routing in transit mode). + // Not built by fromDirectory() and too large to fit in the Kryo cache with 4GB heap. + System.out.println("Building stop-to-vertex distance tables..."); + network.transitLayer.buildDistanceTables(null); + System.out.println("Distance tables built"); + System.out.println("Transport network loaded successfully"); - Javalin app = Javalin.create().start(8003); + HttpServer server = HttpServer.create(new InetSocketAddress(8003), 0); - app.get("/health", ctx -> ctx.result("ok")); + server.createContext("/health", exchange -> { + sendResponse(exchange, 200, "ok"); + }); - app.post("/travel-times", App::handleTravelTimes); + server.createContext("/travel-times", exchange -> { + if (!"POST".equals(exchange.getRequestMethod())) { + sendResponse(exchange, 405, "Method not allowed"); + return; + } + try { + handleTravelTimes(exchange); + } catch (Exception e) { + System.err.println("Error handling travel-times: " + e.getMessage()); + e.printStackTrace(); + sendResponse(exchange, 500, "Internal server error: " + e.getMessage()); + } + }); + server.setExecutor(java.util.concurrent.Executors.newFixedThreadPool(4)); + server.start(); System.out.println("R5 service listening on port 8003"); } - private static void handleTravelTimes(Context ctx) { + private static void sendResponse(HttpExchange exchange, int status, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(bytes); + } + } + + private static void handleTravelTimes(HttpExchange exchange) throws IOException { long t0 = System.currentTimeMillis(); - TravelTimeRequest req = gson.fromJson(ctx.body(), TravelTimeRequest.class); + String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + TravelTimeRequest req = gson.fromJson(body, TravelTimeRequest.class); if (req.origin == null || req.origin.length != 2) { - ctx.status(400).result("origin must be [lat, lon]"); + sendResponse(exchange, 400, "{\"error\":\"origin must be [lat, lon]\"}"); return; } if (req.destinations == null || req.destinations.length == 0) { - ctx.status(400).result("destinations must be non-empty array of [lat, lon]"); + sendResponse(exchange, 400, "{\"error\":\"destinations must be non-empty\"}"); return; } String mode = req.mode != null ? req.mode : "transit"; - // Build destination point set - double[] lats = new double[req.destinations.length]; - double[] lons = new double[req.destinations.length]; + // Build destination point set (Coordinate takes x=lon, y=lat) + Coordinate[] coords = new Coordinate[req.destinations.length]; for (int i = 0; i < req.destinations.length; i++) { - lats[i] = req.destinations[i][0]; - lons[i] = req.destinations[i][1]; + coords[i] = new Coordinate(req.destinations[i][1], req.destinations[i][0]); // lon, lat } - - FreeFormPointSet destinations = new FreeFormPointSet(lats, lons); + FreeFormPointSet destinations = new FreeFormPointSet(coords); // Build the regional task RegionalTask task = new RegionalTask(); @@ -93,7 +139,16 @@ public class App { task.fromLon = req.origin[1]; task.date = LocalDate.now(); task.percentiles = new int[]{50}; - task.monteCarloDraws = 1; + task.recordTimes = true; + task.destinationPointSets = new PointSet[]{ destinations }; + + // Set grid extents from destination point set (required by TravelTimeComputer) + WebMercatorExtents extents = destinations.getWebMercatorExtents(); + task.zoom = extents.zoom; + task.west = extents.west; + task.north = extents.north; + task.width = extents.width; + task.height = extents.height; switch (mode) { case "car": @@ -131,24 +186,31 @@ public class App { task.accessModes = EnumSet.of(LegMode.WALK); task.egressModes = EnumSet.of(LegMode.WALK); task.directModes = EnumSet.of(LegMode.WALK); - task.transitModes = EnumSet.allOf(TransitModes.class); + task.transitModes = EnumSet.of(TransitModes.TRANSIT); break; } // Compute travel times - TravelTimeComputer computer = new TravelTimeComputer(task, network, destinations); - int[][] results = computer.computeTravelTimes(); + TravelTimeComputer computer = new TravelTimeComputer(task, network); + OneOriginResult result = computer.computeTravelTimes(); - // results[percentileIdx][destinationIdx] — we only have 1 percentile (index 0) TravelTimeResponse response = new TravelTimeResponse(); response.travel_times = new double[req.destinations.length]; - int[] times = results[0]; // percentile 0 (the 50th percentile) - for (int i = 0; i < req.destinations.length; i++) { - if (i < times.length && times[i] != Integer.MAX_VALUE) { - response.travel_times[i] = times[i]; // already in minutes - } else { - response.travel_times[i] = -1; // unreachable + TravelTimeResult tt = result.travelTimes; + if (tt != null) { + int[][] values = tt.getValues(); + // values[percentileIndex][destinationIndex] + for (int i = 0; i < req.destinations.length; i++) { + if (i < values[0].length && values[0][i] != Integer.MAX_VALUE) { + response.travel_times[i] = values[0][i]; // already in minutes + } else { + response.travel_times[i] = -1; // unreachable + } + } + } else { + for (int i = 0; i < req.destinations.length; i++) { + response.travel_times[i] = -1; } } @@ -156,6 +218,6 @@ public class App { System.out.println("Travel times (" + mode + ") computed for " + req.destinations.length + " destinations in " + elapsed + "ms"); - ctx.json(response); + sendResponse(exchange, 200, gson.toJson(response)); } } diff --git a/server-rs/src/consts.rs b/server-rs/src/consts.rs index 7af8149..e615e25 100644 --- a/server-rs/src/consts.rs +++ b/server-rs/src/consts.rs @@ -17,3 +17,7 @@ pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02; pub const AREA_SUMMARY_SYSTEM_PROMPT: &str = "You are an experienced estate agent with an expertise in area analysis. Help the user find his/her dream area or perfect postcode to settle in. The user is looking to buy a property based on the filters they provide. Given area statistics, write at most a single concise sentences summarising the key characteristics of the area. Be factual and highlight notable values. Do not use bullet points or headers — just flowing prose. Do not use markdown formatting. Highlight unusual facts that stand out from the average, but do not exaggerate. If there are no notable characteristics, say so. Always write at most a single sentence! Reason about the relation of different statistics to each other."; pub const AREA_SUMMARY_MAX_TOKENS: usize = 300; pub const AREA_SUMMARY_TEMPERATURE: f32 = 0.3; + +pub const AI_FILTERS_SYSTEM_PROMPT: &str = "You are a property search assistant. The user will describe their ideal property or area in natural language. Your job is to translate their description into filter settings. ONLY set filters the user explicitly mentioned or clearly implied. Leave everything else out. Do not guess or add extra filters. If a request is ambiguous, prefer a wider range. Output valid JSON matching the provided schema."; +pub const AI_FILTERS_MAX_TOKENS: usize = 2000; +pub const AI_FILTERS_TEMPERATURE: f32 = 0.0; diff --git a/server-rs/src/data.rs b/server-rs/src/data.rs index 92d8285..d8d1812 100644 --- a/server-rs/src/data.rs +++ b/server-rs/src/data.rs @@ -1,7 +1,9 @@ +mod places; mod poi; mod postcodes; mod property; +pub use places::PlaceData; pub use poi::{POICategoryGroup, POIData}; pub use postcodes::PostcodeData; -pub use property::{precompute_h3, FeatureStats, Histogram, PropertyData}; +pub use property::{precompute_h3, FeatureStats, Histogram, PropertyData, RenovationEvent}; diff --git a/server-rs/src/features.rs b/server-rs/src/features.rs index b032f6d..ffbe192 100644 --- a/server-rs/src/features.rs +++ b/server-rs/src/features.rs @@ -156,6 +156,17 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[ suffix: " rooms", raw: false, }, + FeatureConfig { + name: "Estimated monthly rent", + bounds: Bounds::Percentile { low: 2.0, high: 98.0 }, + step: 25.0, + description: "Median monthly private rent for the local area and bedroom count", + detail: "Median monthly rental price from ONS Private Rental Market Summary Statistics (Oct 2022 - Sep 2023). Matched by local authority district and estimated bedroom count (habitable rooms minus 1). Based on Valuation Office Agency lettings data.", + source: "ons-rental", + prefix: "£", + suffix: "/mo", + raw: false, + }, FeatureConfig { name: "Date of last transaction", bounds: Bounds::Fixed { diff --git a/server-rs/src/routes.rs b/server-rs/src/routes.rs index ebde67d..587d4e3 100644 --- a/server-rs/src/routes.rs +++ b/server-rs/src/routes.rs @@ -1,3 +1,4 @@ +mod ai_filters; mod area_summary; mod export; mod features; @@ -5,6 +6,7 @@ mod hexagon_stats; pub(crate) mod hexagons; mod me; mod pb_proxy; +mod places; mod pois; mod postcode_stats; mod postcodes; @@ -15,6 +17,7 @@ mod stats; mod tiles; pub(crate) mod travel_time; +pub use ai_filters::{build_feature_prompt, build_ollama_schema, post_ai_filters}; pub use area_summary::post_area_summary; pub use export::get_export; pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse}; @@ -22,6 +25,7 @@ pub use hexagon_stats::get_hexagon_stats; pub use hexagons::get_hexagons; pub use me::get_me; pub use pb_proxy::proxy_to_pocketbase; +pub use places::get_places; pub use pois::{get_poi_categories, get_pois}; pub use postcode_stats::get_postcode_stats; pub use postcodes::{get_postcode_lookup, get_postcodes}; diff --git a/server-rs/src/routes/area_summary.rs b/server-rs/src/routes/area_summary.rs index 54b3545..628997a 100644 --- a/server-rs/src/routes/area_summary.rs +++ b/server-rs/src/routes/area_summary.rs @@ -90,7 +90,7 @@ fn build_prompt(req: &AreaSummaryRequest) -> String { } /// Strip `...` blocks from model output -fn strip_think_blocks(text: &str) -> String { +pub(crate) fn strip_think_blocks(text: &str) -> String { let mut result = String::new(); let mut remaining = text; while let Some(start) = remaining.find("") { diff --git a/server-rs/src/routes/export.rs b/server-rs/src/routes/export.rs index 9b3cddc..2767417 100644 --- a/server-rs/src/routes/export.rs +++ b/server-rs/src/routes/export.rs @@ -160,7 +160,8 @@ pub async fn get_export( params.filters.as_deref(), &state.feature_name_to_index, &state.data.enum_values, - ); + ) + .map_err(|err| (StatusCode::BAD_REQUEST, err))?; let public_url = state.public_url.clone(); diff --git a/server-rs/src/routes/features.rs b/server-rs/src/routes/features.rs index 342947b..b99de08 100644 --- a/server-rs/src/routes/features.rs +++ b/server-rs/src/routes/features.rs @@ -48,7 +48,7 @@ pub enum FeatureInfo { #[derive(Clone, Serialize)] pub struct FeatureGroupResponse { - name: String, + pub(crate) name: String, pub(crate) features: Vec, } diff --git a/server-rs/src/routes/hexagons.rs b/server-rs/src/routes/hexagons.rs index aa732ee..ba11733 100644 --- a/server-rs/src/routes/hexagons.rs +++ b/server-rs/src/routes/hexagons.rs @@ -135,7 +135,8 @@ pub async fn get_hexagons( params.filters.as_deref(), &state.feature_name_to_index, &state.data.enum_values, - ); + ) + .map_err(|err| (StatusCode::BAD_REQUEST, err))?; let num_filters = parsed_filters.len() + parsed_enum_filters.len(); let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index); @@ -147,7 +148,7 @@ pub async fn get_hexagons( .map(parse_destination) .transpose() .map_err(|e| (StatusCode::BAD_REQUEST, e))?; - let mode = params.mode.clone().unwrap_or_else(|| "transit".into()); + let mode = params.mode.clone().unwrap_or_else(|| "car".into()); // Capture what we need for the R5 call before moving state into spawn_blocking let r5_url = state.r5_url.clone(); @@ -249,16 +250,16 @@ pub async fn get_hexagons( .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))? .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; - // If a destination was requested and R5 is configured, fetch travel times + // If a destination was requested and R5 is configured, fetch travel times. if let Some(dest) = destination { if r5_url.is_empty() { return Err(( StatusCode::SERVICE_UNAVAILABLE, - "Travel time queries require R5 service (R5_URL not configured)".into(), + "Travel time queries require routing service (R5_URL not configured)".into(), )); } - // Collect hex centroids from the response + // Collect hex centroids let origins: Vec<[f64; 2]> = response .features .iter() @@ -297,8 +298,7 @@ pub async fn get_hexagons( ); } Err(err) => { - warn!("R5 travel time query failed, returning hexagons without travel_time: {}", err); - // Don't fail the whole request — just omit travel_time + warn!("Travel time query failed, returning hexagons without travel_time: {}", err); } } } diff --git a/server-rs/src/routes/places.rs b/server-rs/src/routes/places.rs new file mode 100644 index 0000000..6f44eaf --- /dev/null +++ b/server-rs/src/routes/places.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use axum::extract::Query; +use axum::http::StatusCode; +use axum::response::Json; +use serde::{Deserialize, Serialize}; +use tracing::info; + +use crate::state::AppState; + +#[derive(Serialize)] +pub struct PlaceResult { + name: String, + place_type: String, + lat: f32, + lon: f32, +} + +#[derive(Serialize)] +pub struct PlacesResponse { + places: Vec, +} + +#[derive(Deserialize)] +#[allow(clippy::min_ident_chars)] +pub struct PlacesParams { + q: Option, + limit: Option, +} + +pub async fn get_places( + state: Arc, + Query(params): Query, +) -> Result, (StatusCode, String)> { + let query = params + .q + .filter(|val| !val.is_empty()) + .ok_or((StatusCode::BAD_REQUEST, "Missing 'q' parameter".to_string()))?; + + let limit = params.limit.unwrap_or(7).min(20); + + let places = tokio::task::spawn_blocking(move || { + let t0 = std::time::Instant::now(); + let query_lower = query.to_lowercase(); + let pd = &state.place_data; + + // Linear scan — ~50-100k rows, <1ms + let mut matches: Vec<(usize, bool, u8, usize)> = pd + .name_lower + .iter() + .enumerate() + .filter_map(|(idx, name)| { + if name.contains(&query_lower) { + let is_prefix = name.starts_with(&query_lower); + Some((idx, is_prefix, pd.type_rank[idx], pd.name[idx].len())) + } else { + None + } + }) + .collect(); + + // Sort: prefix first, then by type rank (cities before hamlets), then shorter names first + matches.sort_unstable_by(|lhs, rhs| { + rhs.1 + .cmp(&lhs.1) + .then(lhs.2.cmp(&rhs.2)) + .then(lhs.3.cmp(&rhs.3)) + }); + + matches.truncate(limit); + + let results: Vec = matches + .iter() + .map(|&(idx, ..)| PlaceResult { + name: pd.name[idx].clone(), + place_type: pd.place_type.get(idx).to_string(), + lat: pd.lat[idx], + lon: pd.lon[idx], + }) + .collect(); + + let elapsed = t0.elapsed(); + info!( + query = query.as_str(), + results = results.len(), + scanned = pd.name_lower.len(), + ms = format_args!("{:.1}", elapsed.as_secs_f64() * 1000.0), + "GET /api/places" + ); + + results + }) + .await + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?; + + Ok(Json(PlacesResponse { places })) +} diff --git a/server-rs/src/routes/postcodes.rs b/server-rs/src/routes/postcodes.rs index 1a1fb4f..65c10c6 100644 --- a/server-rs/src/routes/postcodes.rs +++ b/server-rs/src/routes/postcodes.rs @@ -69,7 +69,8 @@ pub async fn get_postcodes( params.filters.as_deref(), &state.feature_name_to_index, &state.data.enum_values, - ); + ) + .map_err(|err| (StatusCode::BAD_REQUEST, err))?; let num_filters = parsed_filters.len() + parsed_enum_filters.len(); let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index); diff --git a/server-rs/src/routes/properties.rs b/server-rs/src/routes/properties.rs index 52a7a89..ce5bcf1 100644 --- a/server-rs/src/routes/properties.rs +++ b/server-rs/src/routes/properties.rs @@ -13,6 +13,7 @@ use crate::parsing::{ cell_for_row, h3_cell_bounds, needs_parent, parse_filters, row_passes_filters, validate_h3_resolution, }; +use crate::data::RenovationEvent; use crate::state::AppState; #[derive(Deserialize)] @@ -41,6 +42,9 @@ pub struct Property { pub is_construction_date_approximate: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub renovation_history: Vec, + #[serde(flatten)] pub features: FxHashMap, } @@ -214,6 +218,7 @@ pub async fn get_hexagon_properties( ), lat: state.data.lat[row], lon: state.data.lon[row], + renovation_history: state.data.renovation_history(row).to_vec(), features, } }) diff --git a/server-rs/src/routes/travel_time.rs b/server-rs/src/routes/travel_time.rs index 322d5b5..a9da19e 100644 --- a/server-rs/src/routes/travel_time.rs +++ b/server-rs/src/routes/travel_time.rs @@ -2,20 +2,25 @@ use serde::{Deserialize, Serialize}; use tracing::warn; #[derive(Serialize)] -struct TravelTimeRequest { - origins: Vec<[f64; 2]>, - destination: [f64; 2], +struct R5Request { + origin: [f64; 2], + destinations: Vec<[f64; 2]>, mode: String, } #[derive(Deserialize)] -struct TravelTimeResponse { - travel_times: Vec>, +struct R5Response { + travel_times: Vec, } -/// Call the R5 service to compute many-to-one travel times. +/// Call the R5 Java service to compute one-to-many travel times. /// -/// Returns a Vec of travel times in minutes (one per origin), with None for unreachable origins. +/// `origins` are hex centroids as `[lat, lon]`. +/// `destination` is the user-chosen point as `[lat, lon]`. +/// `mode` is one of "car", "bicycle", "walking", "transit". +/// +/// R5 computes from destination to all origins (one-to-many from the user's chosen point). +/// Returns a Vec of travel times in minutes (one per origin), with None for unreachable. pub async fn fetch_travel_times( client: &reqwest::Client, r5_url: &str, @@ -23,36 +28,45 @@ pub async fn fetch_travel_times( destination: [f64; 2], mode: &str, ) -> Result>, String> { - let url = format!("{}/travel-times", r5_url); + if origins.is_empty() { + return Ok(vec![]); + } - let request_body = TravelTimeRequest { - origins, - destination, + let body = R5Request { + origin: destination, + destinations: origins, mode: mode.to_string(), }; let resp = client - .post(&url) - .json(&request_body) - .timeout(std::time::Duration::from_secs(60)) + .post(format!("{}/travel-times", r5_url)) + .json(&body) + .timeout(std::time::Duration::from_secs(30)) .send() .await .map_err(|e| { warn!("R5 request failed: {}", e); - format!("R5 service error: {}", e) + format!("R5 routing error: {}", e) })?; if !resp.status().is_success() { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); warn!("R5 returned {}: {}", status, body); - return Err(format!("R5 service returned {}: {}", status, body)); + return Err(format!("R5 returned {}: {}", status, body)); } - let body: TravelTimeResponse = resp.json().await.map_err(|e| { + let r5_resp: R5Response = resp.json().await.map_err(|e| { warn!("Failed to parse R5 response: {}", e); format!("Failed to parse R5 response: {}", e) })?; - Ok(body.travel_times) + // R5 returns -1 for unreachable destinations + let travel_times: Vec> = r5_resp + .travel_times + .into_iter() + .map(|t| if t < 0.0 { None } else { Some(t) }) + .collect(); + + Ok(travel_times) } diff --git a/server-rs/src/state.rs b/server-rs/src/state.rs index 0f90f69..574672a 100644 --- a/server-rs/src/state.rs +++ b/server-rs/src/state.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use rustc_hash::FxHashMap; use crate::auth::TokenCache; -use crate::data::{POICategoryGroup, POIData, PostcodeData, PropertyData}; +use crate::data::{POICategoryGroup, POIData, PlaceData, PostcodeData, PropertyData}; use crate::routes::FeaturesResponse; use crate::utils::GridIndex; @@ -15,6 +15,7 @@ pub struct AppState { pub h3_cells: Vec, pub poi_data: POIData, pub poi_grid: GridIndex, + pub place_data: PlaceData, /// Postcode boundary data for high-zoom rendering pub postcode_data: PostcodeData, /// O(1) lookup: feature name → index in feature_names/feature_data @@ -43,8 +44,12 @@ pub struct AppState { pub ollama_url: String, /// Ollama model name for area summaries (e.g. gemma3:12b) pub ollama_model: String, - /// R5 routing service URL for real-time travel times (empty = disabled) + /// R5 routing service URL for all travel times (empty = disabled) pub r5_url: String, /// Token validation cache (60s TTL) pub token_cache: Arc, + /// JSON schema for Ollama structured output in AI filters + pub ai_filters_schema: serde_json::Value, + /// Feature listing portion of the AI filters prompt + pub ai_filters_feature_prompt: String, } diff --git a/server-rs/src/utils.rs b/server-rs/src/utils.rs index c2bd3f1..5ba9d1a 100644 --- a/server-rs/src/utils.rs +++ b/server-rs/src/utils.rs @@ -1,7 +1,9 @@ mod grid_index; mod hash; mod interned_column; +mod llm; pub use grid_index::GridIndex; pub use hash::{generate_priorities, splitmix64_hash}; pub use interned_column::InternedColumn; +pub use llm::strip_think_blocks; diff --git a/server-rs/src/utils/llm.rs b/server-rs/src/utils/llm.rs new file mode 100644 index 0000000..f75d988 --- /dev/null +++ b/server-rs/src/utils/llm.rs @@ -0,0 +1,15 @@ +/// Strip `...` blocks from model output +pub fn strip_think_blocks(text: &str) -> String { + let mut result = String::new(); + let mut remaining = text; + while let Some(start) = remaining.find("") { + result.push_str(&remaining[..start]); + if let Some(end) = remaining[start..].find("") { + remaining = &remaining[start + end + 8..]; + } else { + return result; + } + } + result.push_str(remaining); + result +}