diff --git a/Taskfile.yml b/Taskfile.yml index 197fbcf..91a050b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -5,16 +5,29 @@ tasks: desc: Install dependencies, generate client, and download data cmds: - uv sync + - cd frontend && npm install + + download: + desc: Download data + deps: + - install + cmds: - uv run python generate_tfl_client.py - uv run python download_land_registry.py - uv run python download_arcgis_data.py - - cd frontend && npm install pipeline: desc: Run data processing pipeline + deps: + - download cmds: - uv run python -m pipeline.run + prepare: + desc: Prepare the application (install, download data, run pipeline) + deps: + - pipeline + server: desc: Run FastAPI backend on port 8001 cmds: diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index bb7fa93..dc9ab53 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,48 +1,77 @@ -import React, { useState, useEffect, useCallback } from 'react'; +import React, { useState, useEffect, useCallback, useRef } from 'react'; import Map from './components/Map'; import Filters from './components/Filters'; import { DEFAULT_FILTERS } from './lib/constants'; +const DEBOUNCE_MS = 150; + export default function App() { const [filters, setFilters] = useState(DEFAULT_FILTERS); const [data, setData] = useState([]); const [resolution, setResolution] = useState(8); + const [bounds, setBounds] = useState(null); const [loading, setLoading] = useState(false); + const debounceRef = useRef(null); + const abortControllerRef = useRef(null); - const fetchData = useCallback(async () => { - setLoading(true); - try { - const params = new URLSearchParams({ - resolution: resolution.toString(), - min_year: filters.minYear.toString(), - max_year: filters.maxYear.toString(), - min_price: filters.minPrice.toString(), - max_price: filters.maxPrice.toString(), - }); - const res = await fetch(`/api/hexagons?${params}`); - const json = await res.json(); - setData( - json.features.map((f) => ({ - h3: f.properties.h3, - ...f.properties, - })) - ); - } catch (err) { - console.error('Failed to fetch data:', err); - } finally { - setLoading(false); - } - }, [filters, resolution]); - + // Debounced fetch when dependencies change useEffect(() => { - fetchData(); - }, [fetchData]); + if (!bounds) return; + + // Clear previous debounce timer + if (debounceRef.current) { + clearTimeout(debounceRef.current); + } + + debounceRef.current = setTimeout(async () => { + // Cancel any in-flight request + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + abortControllerRef.current = new AbortController(); + + setLoading(true); + try { + const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`; + const params = new URLSearchParams({ + resolution: resolution.toString(), + min_year: filters.minYear.toString(), + max_year: filters.maxYear.toString(), + min_price: filters.minPrice.toString(), + max_price: filters.maxPrice.toString(), + bounds: boundsStr, + }); + const res = await fetch(`/api/hexagons?${params}`, { + signal: abortControllerRef.current.signal, + }); + const json = await res.json(); + setData(json.features || []); + } catch (err) { + if (err.name !== 'AbortError') { + console.error('Failed to fetch data:', err); + } + } finally { + setLoading(false); + } + }, DEBOUNCE_MS); + + return () => { + if (debounceRef.current) { + clearTimeout(debounceRef.current); + } + }; + }, [filters, resolution, bounds]); + + const handleViewChange = useCallback(({ resolution: newRes, bounds: newBounds }) => { + setResolution(newRes); + setBounds(newBounds); + }, []); return (
- + {loading && (
Loading... diff --git a/frontend/src/components/Filters.jsx b/frontend/src/components/Filters.jsx index a62f8d6..7572ae9 100644 --- a/frontend/src/components/Filters.jsx +++ b/frontend/src/components/Filters.jsx @@ -54,34 +54,19 @@ export default function Filters({ filters, onChange }) { />
-
-
- - {'< £150k'} -
-
- - £150k - £300k -
-
- - £300k - £500k -
-
- - {'> £500k'} +
+
Average Price
+
+
+ £0 + £200k + £400k + £800k+
diff --git a/frontend/src/components/Map.jsx b/frontend/src/components/Map.jsx index c6cd4dd..f520844 100644 --- a/frontend/src/components/Map.jsx +++ b/frontend/src/components/Map.jsx @@ -1,4 +1,4 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useRef, useEffect, useState, useMemo } from 'react'; import { Map as MapGL } from 'react-map-gl/maplibre'; import DeckGL from '@deck.gl/react'; import { H3HexagonLayer } from '@deck.gl/geo-layers'; @@ -13,32 +13,128 @@ const INITIAL_VIEW = { const MAP_STYLE = 'https://basemaps.cartocdn.com/gl/positron-gl-style/style.json'; +// Continuous color scale from green (low) -> yellow -> red -> purple (high) +const COLOR_SCALE = [ + { price: 0, color: [46, 204, 113] }, // Green + { price: 200000, color: [241, 196, 15] }, // Yellow + { price: 400000, color: [231, 76, 60] }, // Red + { price: 800000, color: [142, 68, 173] }, // Purple +]; + +function interpolateColor(c1, c2, t) { + return [ + Math.round(c1[0] + (c2[0] - c1[0]) * t), + Math.round(c1[1] + (c2[1] - c1[1]) * t), + Math.round(c1[2] + (c2[2] - c1[2]) * t), + ]; +} + function priceToColor(price) { - if (price < 150000) return [46, 204, 113]; - if (price < 300000) return [241, 196, 15]; - if (price < 500000) return [231, 76, 60]; - return [142, 68, 173]; + if (price == null || isNaN(price)) return [128, 128, 128]; // Gray for missing data + + // Clamp to scale range + if (price <= COLOR_SCALE[0].price) return COLOR_SCALE[0].color; + if (price >= COLOR_SCALE[COLOR_SCALE.length - 1].price) { + return COLOR_SCALE[COLOR_SCALE.length - 1].color; + } + + // Find the two colors to interpolate between + for (let i = 0; i < COLOR_SCALE.length - 1; i++) { + const lower = COLOR_SCALE[i]; + const upper = COLOR_SCALE[i + 1]; + if (price >= lower.price && price <= upper.price) { + const t = (price - lower.price) / (upper.price - lower.price); + return interpolateColor(lower.color, upper.color, t); + } + } + + return COLOR_SCALE[COLOR_SCALE.length - 1].color; } function zoomToResolution(zoom) { - if (zoom < 7) return 6; + if (zoom < 8) return 6; if (zoom < 9) return 7; if (zoom < 11) return 8; - if (zoom < 13) return 9; - if (zoom < 15) return 10; - if (zoom < 17) return 11; - return 12; + if (zoom < 14) return 9; + if (zoom < 16) return 10; + return 11; } -export default function Map({ data, onZoom }) { - const onViewStateChange = useCallback( - ({ viewState }) => { - onZoom(zoomToResolution(viewState.zoom)); - }, - [onZoom] - ); +function getBoundsFromViewState(viewState, width, height) { + const { longitude, latitude, zoom } = viewState; - const layers = [ + // Clamp latitude to valid Mercator range to avoid math errors + const clampedLat = Math.max(-85, Math.min(85, latitude)); + + // Web Mercator projection math + const TILE_SIZE = 256; + const scale = Math.pow(2, zoom); + const worldSize = TILE_SIZE * scale; + + // Longitude is linear + const degreesPerPixelLng = 360 / worldSize; + const halfWidthDeg = (width / 2) * degreesPerPixelLng; + + // Latitude uses Mercator projection (non-linear) + // Convert center lat to pixel y, offset by half height, convert back to lat + const latRad = clampedLat * Math.PI / 180; + const mercatorY = (1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2; + const centerPixelY = mercatorY * worldSize; + + const topPixelY = centerPixelY - height / 2; + const bottomPixelY = centerPixelY + height / 2; + + // Convert pixel Y back to latitude + const pixelYToLat = (pixelY) => { + const mercY = Math.max(0.001, Math.min(0.999, pixelY / worldSize)); // Clamp to avoid edge cases + const latRadians = Math.atan(Math.sinh(Math.PI * (1 - 2 * mercY))); + return latRadians * 180 / Math.PI; + }; + + const north = Math.min(85, pixelYToLat(topPixelY)); + const south = Math.max(-85, pixelYToLat(bottomPixelY)); + const west = Math.max(-180, longitude - halfWidthDeg); + const east = Math.min(180, longitude + halfWidthDeg); + + return { south, west, north, east }; +} + +export default function Map({ data, onViewChange }) { + const containerRef = useRef(null); + const [viewState, setViewState] = useState(INITIAL_VIEW); + const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); + + // Track container dimensions with ResizeObserver + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const observer = new ResizeObserver((entries) => { + const { width, height } = entries[0].contentRect; + if (width > 0 && height > 0) { + setDimensions({ width, height }); + } + }); + + observer.observe(container); + return () => observer.disconnect(); + }, []); + + // Notify parent when view or dimensions change + useEffect(() => { + if (dimensions.width === 0 || dimensions.height === 0) return; + + const bounds = getBoundsFromViewState(viewState, dimensions.width, dimensions.height); + const resolution = zoomToResolution(viewState.zoom); + + onViewChange({ resolution, bounds }); + }, [viewState, dimensions, onViewChange]); + + const handleViewStateChange = useCallback(({ viewState: newViewState }) => { + setViewState(newViewState); + }, []); + + const layers = useMemo(() => [ new H3HexagonLayer({ id: 'h3-hexagons', data, @@ -48,15 +144,15 @@ export default function Map({ data, onZoom }) { pickable: true, opacity: 0.7, }), - ]; + ], [data]); return ( -
+
diff --git a/frontend/src/lib/constants.js b/frontend/src/lib/constants.js index a81b1f2..c38b636 100644 --- a/frontend/src/lib/constants.js +++ b/frontend/src/lib/constants.js @@ -1,11 +1,12 @@ // Filter configuration constants +// Should match backend pipeline/config.py export const YEAR_MIN = 1995; export const YEAR_MAX = 2024; export const YEAR_STEP = 1; export const PRICE_MIN = 0; -export const PRICE_MAX = 2000000; +export const PRICE_MAX = 5000000; // £5M max for slider, but no server-side cap export const PRICE_STEP = 50000; export const DEFAULT_FILTERS = { diff --git a/pipeline/config.py b/pipeline/config.py index b30cd89..4c18b2e 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -9,7 +9,7 @@ AGGREGATES_DIR = PROCESSED_DIR / "aggregates" # H3 resolutions to generate and serve # https://h3geo.org/docs/core-library/restable/#average-area-in-m2 -H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11, 12] +H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11] DEFAULT_H3_RESOLUTION = 8 # Year filters @@ -20,4 +20,4 @@ DEFAULT_MAX_YEAR = 2024 # Price filters DEFAULT_MIN_PRICE = 0 -DEFAULT_MAX_PRICE = 2_000_000 +DEFAULT_MAX_PRICE = 100_000_000 diff --git a/pipeline/run.py b/pipeline/run.py index cd6ba07..8bb3884 100644 --- a/pipeline/run.py +++ b/pipeline/run.py @@ -1,10 +1,8 @@ """Pipeline CLI to process property data with H3 spatial indexing.""" -from pathlib import Path import polars as pl -from tqdm import tqdm -from pipeline.sources.postcodes import save_postcodes, DATA_DIR +from pipeline.sources.postcodes import save_postcodes from pipeline.sources.property_prices import PropertyPricesSource from pipeline.processors.h3_aggregator import save_aggregates diff --git a/pipeline/sources/postcodes.py b/pipeline/sources/postcodes.py index 1364ea3..d0105b5 100644 --- a/pipeline/sources/postcodes.py +++ b/pipeline/sources/postcodes.py @@ -28,7 +28,8 @@ def process_postcodes() -> pl.LazyFrame: df = df.with_columns( pl.struct(["lat", "long"]) .map_elements( - lambda x: lat_long_to_h3(x["lat"], x["long"], res), + # Capture res by value using default argument to avoid closure bug + lambda x, res=res: lat_long_to_h3(x["lat"], x["long"], res), return_dtype=pl.Utf8, ) .alias(col_name) diff --git a/server/main.py b/server/main.py index 46e738a..14cf941 100644 --- a/server/main.py +++ b/server/main.py @@ -10,7 +10,7 @@ app = FastAPI(title="Property Map API") app.add_middleware( CORSMiddleware, allow_origins=["*"], - allow_credentials=True, + allow_credentials=False, # Cannot use True with wildcard origins allow_methods=["*"], allow_headers=["*"], ) diff --git a/server/routes/hexagons.py b/server/routes/hexagons.py index 04f3352..592b097 100644 --- a/server/routes/hexagons.py +++ b/server/routes/hexagons.py @@ -1,5 +1,5 @@ from typing import Any -from fastapi import APIRouter, Query +from fastapi import APIRouter, Query, HTTPException import polars as pl import h3 @@ -16,19 +16,41 @@ from server.config import ( router = APIRouter() -def h3_to_geojson_feature(h3_index: str, properties: dict[str, Any]) -> dict: - """Convert H3 index to GeoJSON feature with polygon geometry.""" - boundary = h3.cell_to_boundary(h3_index) - # h3 returns (lat, lng) pairs, GeoJSON needs [lng, lat] - coordinates = [[lng, lat] for lat, lng in boundary] - # Close the polygon - coordinates.append(coordinates[0]) +def get_h3_cells_for_bounds( + south: float, west: float, north: float, east: float, resolution: int +) -> set[str] | None: + """Get all H3 cells that cover a bounding box. Returns None if area too large.""" + # Clamp to valid ranges + south = max(-85, min(85, south)) + north = max(-85, min(85, north)) + west = max(-180, min(180, west)) + east = max(-180, min(180, east)) - return { - "type": "Feature", - "properties": {"h3": h3_index, **properties}, - "geometry": {"type": "Polygon", "coordinates": [coordinates]}, - } + # Ensure valid bounds + if south >= north or west >= east: + return set() + + # If viewport is too large, return None to skip filtering + # This prevents H3 from trying to enumerate millions of cells + lat_span = north - south + lng_span = east - west + if lat_span > 20 or lng_span > 30: + return None + + # Create polygon from bounds (counter-clockwise winding for H3/GeoJSON) + # Order: SW -> NW -> NE -> SE -> SW + polygon = [ + (south, west), + (north, west), + (north, east), + (south, east), + (south, west), + ] + + try: + return h3.polygon_to_cells(h3.LatLngPoly(polygon), resolution) + except Exception: + return None @router.get("/hexagons") @@ -44,20 +66,41 @@ async def get_hexagons( min_price: float = Query(DEFAULT_MIN_PRICE, description="Minimum average price"), max_price: float = Query(DEFAULT_MAX_PRICE, description="Maximum average price"), bounds: str | None = Query( - None, description="Bounding box: lat1,lng1,lat2,lng2" + None, description="Bounding box: south,west,north,east" ), ) -> dict: - """Get aggregated property data as GeoJSON hexagons.""" + """Get aggregated property data as GeoJSON hexagons within bounds.""" if resolution not in VALID_RESOLUTIONS: resolution = DEFAULT_RESOLUTION + # Bounds are required for efficient queries + if not bounds: + raise HTTPException(status_code=400, detail="bounds parameter is required") + + # Parse bounds + try: + south, west, north, east = map(float, bounds.split(",")) + except ValueError: + raise HTTPException( + status_code=400, detail="Invalid bounds format. Use: south,west,north,east" + ) + # Load the appropriate resolution file parquet_path = AGGREGATES_DIR / f"res{resolution}.parquet" if not parquet_path.exists(): - return {"type": "FeatureCollection", "features": []} + return {"features": []} df = pl.scan_parquet(parquet_path) + # Get H3 cells that cover the viewport (None if too large to enumerate) + viewport_cells = get_h3_cells_for_bounds(south, west, north, east, resolution) + + # Filter to only cells in viewport (skip if viewport too large) + if viewport_cells is not None: + if len(viewport_cells) == 0: + return {"features": []} + df = df.filter(pl.col("h3").is_in(viewport_cells)) + # Filter by year range df = df.filter((pl.col("year") >= min_year) & (pl.col("year") <= max_year)) @@ -80,19 +123,27 @@ async def get_hexagons( (pl.col("avg_price") >= min_price) & (pl.col("avg_price") <= max_price) ) - # Collect and convert to GeoJSON + # Limit results to prevent browser crashes + MAX_HEXAGONS = 50000 + df = df.limit(MAX_HEXAGONS) + + # Collect results result = df.collect() - features = [] - for row in result.iter_rows(named=True): - h3_index = row["h3"] - properties = { + # Return lightweight response - just h3 index and properties + # Frontend H3HexagonLayer will render the geometry + # Use to_dicts() which is faster than iter_rows for large results + rows = result.to_dicts() + features = [ + { + "h3": row["h3"], "count": row["count"], "avg_price": round(row["avg_price"], 2), "median_price": round(row["median_price"], 2) if row["median_price"] else None, "min_price": row["min_price"], "max_price": row["max_price"], } - features.append(h3_to_geojson_feature(h3_index, properties)) + for row in rows + ] - return {"type": "FeatureCollection", "features": features} + return {"features": features, "truncated": len(rows) >= MAX_HEXAGONS}