Improve map

Taskfile.yml

@@ -5,16 +5,29 @@ tasks:
     desc: Install dependencies, generate client, and download data
     cmds:
       - uv sync
+      - cd frontend && npm install
+
+  download:
+    desc: Download data
+    deps:
+      - install
+    cmds:
       - uv run python generate_tfl_client.py
       - uv run python download_land_registry.py
       - uv run python download_arcgis_data.py
-      - cd frontend && npm install
 
   pipeline:
     desc: Run data processing pipeline
+    deps:
+      - download
     cmds:
       - uv run python -m pipeline.run
 
+  prepare:
+    desc: Prepare the application (install, download data, run pipeline)
+    deps:
+      - pipeline
+
   server:
     desc: Run FastAPI backend on port 8001
     cmds:

frontend/src/App.jsx

@@ -1,48 +1,77 @@
-import React, { useState, useEffect, useCallback } from 'react';
+import React, { useState, useEffect, useCallback, useRef } from 'react';
 import Map from './components/Map';
 import Filters from './components/Filters';
 import { DEFAULT_FILTERS } from './lib/constants';
 
+const DEBOUNCE_MS = 150;
+
 export default function App() {
   const [filters, setFilters] = useState(DEFAULT_FILTERS);
   const [data, setData] = useState([]);
   const [resolution, setResolution] = useState(8);
+  const [bounds, setBounds] = useState(null);
   const [loading, setLoading] = useState(false);
+  const debounceRef = useRef(null);
+  const abortControllerRef = useRef(null);
 
-  const fetchData = useCallback(async () => {
+  // Debounced fetch when dependencies change
-    setLoading(true);
-    try {
-      const params = new URLSearchParams({
-        resolution: resolution.toString(),
-        min_year: filters.minYear.toString(),
-        max_year: filters.maxYear.toString(),
-        min_price: filters.minPrice.toString(),
-        max_price: filters.maxPrice.toString(),
-      });
-      const res = await fetch(`/api/hexagons?${params}`);
-      const json = await res.json();
-      setData(
-        json.features.map((f) => ({
-          h3: f.properties.h3,
-          ...f.properties,
-        }))
-      );
-    } catch (err) {
-      console.error('Failed to fetch data:', err);
-    } finally {
-      setLoading(false);
-    }
-  }, [filters, resolution]);
-
   useEffect(() => {
-    fetchData();
+    if (!bounds) return;
-  }, [fetchData]);
+
+    // Clear previous debounce timer
+    if (debounceRef.current) {
+      clearTimeout(debounceRef.current);
+    }
+
+    debounceRef.current = setTimeout(async () => {
+      // Cancel any in-flight request
+      if (abortControllerRef.current) {
+        abortControllerRef.current.abort();
+      }
+      abortControllerRef.current = new AbortController();
+
+      setLoading(true);
+      try {
+        const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`;
+        const params = new URLSearchParams({
+          resolution: resolution.toString(),
+          min_year: filters.minYear.toString(),
+          max_year: filters.maxYear.toString(),
+          min_price: filters.minPrice.toString(),
+          max_price: filters.maxPrice.toString(),
+          bounds: boundsStr,
+        });
+        const res = await fetch(`/api/hexagons?${params}`, {
+          signal: abortControllerRef.current.signal,
+        });
+        const json = await res.json();
+        setData(json.features || []);
+      } catch (err) {
+        if (err.name !== 'AbortError') {
+          console.error('Failed to fetch data:', err);
+        }
+      } finally {
+        setLoading(false);
+      }
+    }, DEBOUNCE_MS);
+
+    return () => {
+      if (debounceRef.current) {
+        clearTimeout(debounceRef.current);
+      }
+    };
+  }, [filters, resolution, bounds]);
+
+  const handleViewChange = useCallback(({ resolution: newRes, bounds: newBounds }) => {
+    setResolution(newRes);
+    setBounds(newBounds);
+  }, []);
 
   return (
     <div className="h-screen flex">
       <Filters filters={filters} onChange={setFilters} />
       <div className="flex-1 relative">
-        <Map data={data} onZoom={setResolution} />
+        <Map data={data} onViewChange={handleViewChange} />
         {loading && (
           <div className="absolute top-4 right-4 bg-white px-3 py-1 rounded shadow">
             Loading...

frontend/src/components/Filters.jsx

@@ -54,34 +54,19 @@ export default function Filters({ filters, onChange }) {
         />
       </div>
 
-      <div className="mt-6 p-3 bg-slate-100 rounded text-xs space-y-1">
+      <div className="mt-6 p-3 bg-slate-100 rounded text-xs">
-        <div className="flex items-center gap-2">
+        <div className="mb-2 font-medium">Average Price</div>
-          <span
+        <div
-            className="w-3 h-3 rounded"
+          className="h-4 rounded"
-            style={{ backgroundColor: 'rgb(46, 204, 113)' }}
+          style={{
-          ></span>
+            background: 'linear-gradient(to right, rgb(46, 204, 113), rgb(241, 196, 15), rgb(231, 76, 60), rgb(142, 68, 173))',
-          <span>{'< £150k'}</span>
+          }}
-        </div>
+        ></div>
-        <div className="flex items-center gap-2">
+        <div className="flex justify-between mt-1">
-          <span
+          <span>£0</span>
-            className="w-3 h-3 rounded"
+          <span>£200k</span>
-            style={{ backgroundColor: 'rgb(241, 196, 15)' }}
+          <span>£400k</span>
-          ></span>
+          <span>£800k+</span>
-          <span>£150k - £300k</span>
-        </div>
-        <div className="flex items-center gap-2">
-          <span
-            className="w-3 h-3 rounded"
-            style={{ backgroundColor: 'rgb(231, 76, 60)' }}
-          ></span>
-          <span>£300k - £500k</span>
-        </div>
-        <div className="flex items-center gap-2">
-          <span
-            className="w-3 h-3 rounded"
-            style={{ backgroundColor: 'rgb(142, 68, 173)' }}
-          ></span>
-          <span>{'> £500k'}</span>
         </div>
       </div>
     </div>

frontend/src/components/Map.jsx

@@ -1,4 +1,4 @@
-import React, { useCallback } from 'react';
+import React, { useCallback, useRef, useEffect, useState, useMemo } from 'react';
 import { Map as MapGL } from 'react-map-gl/maplibre';
 import DeckGL from '@deck.gl/react';
 import { H3HexagonLayer } from '@deck.gl/geo-layers';
@@ -13,32 +13,128 @@ const INITIAL_VIEW = {
 
 const MAP_STYLE = 'https://basemaps.cartocdn.com/gl/positron-gl-style/style.json';
 
+// Continuous color scale from green (low) -> yellow -> red -> purple (high)
+const COLOR_SCALE = [
+  { price: 0, color: [46, 204, 113] },        // Green
+  { price: 200000, color: [241, 196, 15] },   // Yellow
+  { price: 400000, color: [231, 76, 60] },    // Red
+  { price: 800000, color: [142, 68, 173] },   // Purple
+];
+
+function interpolateColor(c1, c2, t) {
+  return [
+    Math.round(c1[0] + (c2[0] - c1[0]) * t),
+    Math.round(c1[1] + (c2[1] - c1[1]) * t),
+    Math.round(c1[2] + (c2[2] - c1[2]) * t),
+  ];
+}
+
 function priceToColor(price) {
-  if (price < 150000) return [46, 204, 113];
+  if (price == null || isNaN(price)) return [128, 128, 128]; // Gray for missing data
-  if (price < 300000) return [241, 196, 15];
+
-  if (price < 500000) return [231, 76, 60];
+  // Clamp to scale range
-  return [142, 68, 173];
+  if (price <= COLOR_SCALE[0].price) return COLOR_SCALE[0].color;
+  if (price >= COLOR_SCALE[COLOR_SCALE.length - 1].price) {
+    return COLOR_SCALE[COLOR_SCALE.length - 1].color;
+  }
+
+  // Find the two colors to interpolate between
+  for (let i = 0; i < COLOR_SCALE.length - 1; i++) {
+    const lower = COLOR_SCALE[i];
+    const upper = COLOR_SCALE[i + 1];
+    if (price >= lower.price && price <= upper.price) {
+      const t = (price - lower.price) / (upper.price - lower.price);
+      return interpolateColor(lower.color, upper.color, t);
+    }
+  }
+
+  return COLOR_SCALE[COLOR_SCALE.length - 1].color;
 }
 
 function zoomToResolution(zoom) {
-  if (zoom < 7) return 6;
+  if (zoom < 8) return 6;
   if (zoom < 9) return 7;
   if (zoom < 11) return 8;
-  if (zoom < 13) return 9;
+  if (zoom < 14) return 9;
-  if (zoom < 15) return 10;
+  if (zoom < 16) return 10;
-  if (zoom < 17) return 11;
+  return 11;
-  return 12;
 }
 
-export default function Map({ data, onZoom }) {
+function getBoundsFromViewState(viewState, width, height) {
-  const onViewStateChange = useCallback(
+  const { longitude, latitude, zoom } = viewState;
-    ({ viewState }) => {
-      onZoom(zoomToResolution(viewState.zoom));
-    },
-    [onZoom]
-  );
 
-  const layers = [
+  // Clamp latitude to valid Mercator range to avoid math errors
+  const clampedLat = Math.max(-85, Math.min(85, latitude));
+
+  // Web Mercator projection math
+  const TILE_SIZE = 256;
+  const scale = Math.pow(2, zoom);
+  const worldSize = TILE_SIZE * scale;
+
+  // Longitude is linear
+  const degreesPerPixelLng = 360 / worldSize;
+  const halfWidthDeg = (width / 2) * degreesPerPixelLng;
+
+  // Latitude uses Mercator projection (non-linear)
+  // Convert center lat to pixel y, offset by half height, convert back to lat
+  const latRad = clampedLat * Math.PI / 180;
+  const mercatorY = (1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2;
+  const centerPixelY = mercatorY * worldSize;
+
+  const topPixelY = centerPixelY - height / 2;
+  const bottomPixelY = centerPixelY + height / 2;
+
+  // Convert pixel Y back to latitude
+  const pixelYToLat = (pixelY) => {
+    const mercY = Math.max(0.001, Math.min(0.999, pixelY / worldSize)); // Clamp to avoid edge cases
+    const latRadians = Math.atan(Math.sinh(Math.PI * (1 - 2 * mercY)));
+    return latRadians * 180 / Math.PI;
+  };
+
+  const north = Math.min(85, pixelYToLat(topPixelY));
+  const south = Math.max(-85, pixelYToLat(bottomPixelY));
+  const west = Math.max(-180, longitude - halfWidthDeg);
+  const east = Math.min(180, longitude + halfWidthDeg);
+
+  return { south, west, north, east };
+}
+
+export default function Map({ data, onViewChange }) {
+  const containerRef = useRef(null);
+  const [viewState, setViewState] = useState(INITIAL_VIEW);
+  const [dimensions, setDimensions] = useState({ width: 0, height: 0 });
+
+  // Track container dimensions with ResizeObserver
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container) return;
+
+    const observer = new ResizeObserver((entries) => {
+      const { width, height } = entries[0].contentRect;
+      if (width > 0 && height > 0) {
+        setDimensions({ width, height });
+      }
+    });
+
+    observer.observe(container);
+    return () => observer.disconnect();
+  }, []);
+
+  // Notify parent when view or dimensions change
+  useEffect(() => {
+    if (dimensions.width === 0 || dimensions.height === 0) return;
+
+    const bounds = getBoundsFromViewState(viewState, dimensions.width, dimensions.height);
+    const resolution = zoomToResolution(viewState.zoom);
+
+    onViewChange({ resolution, bounds });
+  }, [viewState, dimensions, onViewChange]);
+
+  const handleViewStateChange = useCallback(({ viewState: newViewState }) => {
+    setViewState(newViewState);
+  }, []);
+
+  const layers = useMemo(() => [
     new H3HexagonLayer({
       id: 'h3-hexagons',
       data,
@@ -48,15 +144,15 @@ export default function Map({ data, onZoom }) {
       pickable: true,
       opacity: 0.7,
     }),
-  ];
+  ], [data]);
 
   return (
-    <div className="flex-1">
+    <div className="flex-1 h-full" ref={containerRef}>
       <DeckGL
-        initialViewState={INITIAL_VIEW}
+        viewState={viewState}
         controller
         layers={layers}
-        onViewStateChange={onViewStateChange}
+        onViewStateChange={handleViewStateChange}
       >
         <MapGL mapStyle={MAP_STYLE} />
       </DeckGL>

frontend/src/lib/constants.js

@@ -1,11 +1,12 @@
 // Filter configuration constants
+// Should match backend pipeline/config.py
 
 export const YEAR_MIN = 1995;
 export const YEAR_MAX = 2024;
 export const YEAR_STEP = 1;
 
 export const PRICE_MIN = 0;
-export const PRICE_MAX = 2000000;
+export const PRICE_MAX = 5000000; // £5M max for slider, but no server-side cap
 export const PRICE_STEP = 50000;
 
 export const DEFAULT_FILTERS = {

pipeline/config.py

@@ -9,7 +9,7 @@ AGGREGATES_DIR = PROCESSED_DIR / "aggregates"
 
 # H3 resolutions to generate and serve
 # https://h3geo.org/docs/core-library/restable/#average-area-in-m2
-H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11, 12]
+H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11]
 DEFAULT_H3_RESOLUTION = 8
 
 # Year filters
@@ -20,4 +20,4 @@ DEFAULT_MAX_YEAR = 2024
 
 # Price filters
 DEFAULT_MIN_PRICE = 0
-DEFAULT_MAX_PRICE = 2_000_000
+DEFAULT_MAX_PRICE = 100_000_000

pipeline/run.py

@@ -1,10 +1,8 @@
 """Pipeline CLI to process property data with H3 spatial indexing."""
 
-from pathlib import Path
 import polars as pl
-from tqdm import tqdm
 
-from pipeline.sources.postcodes import save_postcodes, DATA_DIR
+from pipeline.sources.postcodes import save_postcodes
 from pipeline.sources.property_prices import PropertyPricesSource
 from pipeline.processors.h3_aggregator import save_aggregates
 

pipeline/sources/postcodes.py

@@ -28,7 +28,8 @@ def process_postcodes() -> pl.LazyFrame:
         df = df.with_columns(
             pl.struct(["lat", "long"])
             .map_elements(
-                lambda x: lat_long_to_h3(x["lat"], x["long"], res),
+                # Capture res by value using default argument to avoid closure bug
+                lambda x, res=res: lat_long_to_h3(x["lat"], x["long"], res),
                 return_dtype=pl.Utf8,
             )
             .alias(col_name)

server/main.py

@@ -10,7 +10,7 @@ app = FastAPI(title="Property Map API")
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
-    allow_credentials=True,
+    allow_credentials=False,  # Cannot use True with wildcard origins
     allow_methods=["*"],
     allow_headers=["*"],
 )

server/routes/hexagons.py

@@ -1,5 +1,5 @@
 from typing import Any
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, Query, HTTPException
 import polars as pl
 import h3
 
@@ -16,19 +16,41 @@ from server.config import (
 router = APIRouter()
 
 
-def h3_to_geojson_feature(h3_index: str, properties: dict[str, Any]) -> dict:
+def get_h3_cells_for_bounds(
-    """Convert H3 index to GeoJSON feature with polygon geometry."""
+    south: float, west: float, north: float, east: float, resolution: int
-    boundary = h3.cell_to_boundary(h3_index)
+) -> set[str] | None:
-    # h3 returns (lat, lng) pairs, GeoJSON needs [lng, lat]
+    """Get all H3 cells that cover a bounding box. Returns None if area too large."""
-    coordinates = [[lng, lat] for lat, lng in boundary]
+    # Clamp to valid ranges
-    # Close the polygon
+    south = max(-85, min(85, south))
-    coordinates.append(coordinates[0])
+    north = max(-85, min(85, north))
+    west = max(-180, min(180, west))
+    east = max(-180, min(180, east))
 
-    return {
+    # Ensure valid bounds
-        "type": "Feature",
+    if south >= north or west >= east:
-        "properties": {"h3": h3_index, **properties},
+        return set()
-        "geometry": {"type": "Polygon", "coordinates": [coordinates]},
+
-    }
+    # If viewport is too large, return None to skip filtering
+    # This prevents H3 from trying to enumerate millions of cells
+    lat_span = north - south
+    lng_span = east - west
+    if lat_span > 20 or lng_span > 30:
+        return None
+
+    # Create polygon from bounds (counter-clockwise winding for H3/GeoJSON)
+    # Order: SW -> NW -> NE -> SE -> SW
+    polygon = [
+        (south, west),
+        (north, west),
+        (north, east),
+        (south, east),
+        (south, west),
+    ]
+
+    try:
+        return h3.polygon_to_cells(h3.LatLngPoly(polygon), resolution)
+    except Exception:
+        return None
 
 
 @router.get("/hexagons")
@@ -44,20 +66,41 @@ async def get_hexagons(
     min_price: float = Query(DEFAULT_MIN_PRICE, description="Minimum average price"),
     max_price: float = Query(DEFAULT_MAX_PRICE, description="Maximum average price"),
     bounds: str | None = Query(
-        None, description="Bounding box: lat1,lng1,lat2,lng2"
+        None, description="Bounding box: south,west,north,east"
     ),
 ) -> dict:
-    """Get aggregated property data as GeoJSON hexagons."""
+    """Get aggregated property data as GeoJSON hexagons within bounds."""
     if resolution not in VALID_RESOLUTIONS:
         resolution = DEFAULT_RESOLUTION
 
+    # Bounds are required for efficient queries
+    if not bounds:
+        raise HTTPException(status_code=400, detail="bounds parameter is required")
+
+    # Parse bounds
+    try:
+        south, west, north, east = map(float, bounds.split(","))
+    except ValueError:
+        raise HTTPException(
+            status_code=400, detail="Invalid bounds format. Use: south,west,north,east"
+        )
+
     # Load the appropriate resolution file
     parquet_path = AGGREGATES_DIR / f"res{resolution}.parquet"
     if not parquet_path.exists():
-        return {"type": "FeatureCollection", "features": []}
+        return {"features": []}
 
     df = pl.scan_parquet(parquet_path)
 
+    # Get H3 cells that cover the viewport (None if too large to enumerate)
+    viewport_cells = get_h3_cells_for_bounds(south, west, north, east, resolution)
+
+    # Filter to only cells in viewport (skip if viewport too large)
+    if viewport_cells is not None:
+        if len(viewport_cells) == 0:
+            return {"features": []}
+        df = df.filter(pl.col("h3").is_in(viewport_cells))
+
     # Filter by year range
     df = df.filter((pl.col("year") >= min_year) & (pl.col("year") <= max_year))
 
@@ -80,19 +123,27 @@ async def get_hexagons(
         (pl.col("avg_price") >= min_price) & (pl.col("avg_price") <= max_price)
     )
 
-    # Collect and convert to GeoJSON
+    # Limit results to prevent browser crashes
+    MAX_HEXAGONS = 50000
+    df = df.limit(MAX_HEXAGONS)
+
+    # Collect results
     result = df.collect()
 
-    features = []
+    # Return lightweight response - just h3 index and properties
-    for row in result.iter_rows(named=True):
+    # Frontend H3HexagonLayer will render the geometry
-        h3_index = row["h3"]
+    # Use to_dicts() which is faster than iter_rows for large results
-        properties = {
+    rows = result.to_dicts()
+    features = [
+        {
+            "h3": row["h3"],
             "count": row["count"],
             "avg_price": round(row["avg_price"], 2),
             "median_price": round(row["median_price"], 2) if row["median_price"] else None,
             "min_price": row["min_price"],
             "max_price": row["max_price"],
         }
-        features.append(h3_to_geojson_feature(h3_index, properties))
+        for row in rows
+    ]
 
-    return {"type": "FeatureCollection", "features": features}
+    return {"features": features, "truncated": len(rows) >= MAX_HEXAGONS}