Improve map
This commit is contained in:
parent
ced6b16140
commit
a2e4c29839
10 changed files with 285 additions and 111 deletions
15
Taskfile.yml
15
Taskfile.yml
|
|
@ -5,16 +5,29 @@ tasks:
|
|||
desc: Install dependencies, generate client, and download data
|
||||
cmds:
|
||||
- uv sync
|
||||
- cd frontend && npm install
|
||||
|
||||
download:
|
||||
desc: Download data
|
||||
deps:
|
||||
- install
|
||||
cmds:
|
||||
- uv run python generate_tfl_client.py
|
||||
- uv run python download_land_registry.py
|
||||
- uv run python download_arcgis_data.py
|
||||
- cd frontend && npm install
|
||||
|
||||
pipeline:
|
||||
desc: Run data processing pipeline
|
||||
deps:
|
||||
- download
|
||||
cmds:
|
||||
- uv run python -m pipeline.run
|
||||
|
||||
prepare:
|
||||
desc: Prepare the application (install, download data, run pipeline)
|
||||
deps:
|
||||
- pipeline
|
||||
|
||||
server:
|
||||
desc: Run FastAPI backend on port 8001
|
||||
cmds:
|
||||
|
|
|
|||
|
|
@ -1,48 +1,77 @@
|
|||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
import React, { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import Map from './components/Map';
|
||||
import Filters from './components/Filters';
|
||||
import { DEFAULT_FILTERS } from './lib/constants';
|
||||
|
||||
const DEBOUNCE_MS = 150;
|
||||
|
||||
export default function App() {
|
||||
const [filters, setFilters] = useState(DEFAULT_FILTERS);
|
||||
const [data, setData] = useState([]);
|
||||
const [resolution, setResolution] = useState(8);
|
||||
const [bounds, setBounds] = useState(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const debounceRef = useRef(null);
|
||||
const abortControllerRef = useRef(null);
|
||||
|
||||
// Debounced fetch when dependencies change
|
||||
useEffect(() => {
|
||||
if (!bounds) return;
|
||||
|
||||
// Clear previous debounce timer
|
||||
if (debounceRef.current) {
|
||||
clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
debounceRef.current = setTimeout(async () => {
|
||||
// Cancel any in-flight request
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
}
|
||||
abortControllerRef.current = new AbortController();
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`;
|
||||
const params = new URLSearchParams({
|
||||
resolution: resolution.toString(),
|
||||
min_year: filters.minYear.toString(),
|
||||
max_year: filters.maxYear.toString(),
|
||||
min_price: filters.minPrice.toString(),
|
||||
max_price: filters.maxPrice.toString(),
|
||||
bounds: boundsStr,
|
||||
});
|
||||
const res = await fetch(`/api/hexagons?${params}`, {
|
||||
signal: abortControllerRef.current.signal,
|
||||
});
|
||||
const res = await fetch(`/api/hexagons?${params}`);
|
||||
const json = await res.json();
|
||||
setData(
|
||||
json.features.map((f) => ({
|
||||
h3: f.properties.h3,
|
||||
...f.properties,
|
||||
}))
|
||||
);
|
||||
setData(json.features || []);
|
||||
} catch (err) {
|
||||
if (err.name !== 'AbortError') {
|
||||
console.error('Failed to fetch data:', err);
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [filters, resolution]);
|
||||
}, DEBOUNCE_MS);
|
||||
|
||||
useEffect(() => {
|
||||
fetchData();
|
||||
}, [fetchData]);
|
||||
return () => {
|
||||
if (debounceRef.current) {
|
||||
clearTimeout(debounceRef.current);
|
||||
}
|
||||
};
|
||||
}, [filters, resolution, bounds]);
|
||||
|
||||
const handleViewChange = useCallback(({ resolution: newRes, bounds: newBounds }) => {
|
||||
setResolution(newRes);
|
||||
setBounds(newBounds);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="h-screen flex">
|
||||
<Filters filters={filters} onChange={setFilters} />
|
||||
<div className="flex-1 relative">
|
||||
<Map data={data} onZoom={setResolution} />
|
||||
<Map data={data} onViewChange={handleViewChange} />
|
||||
{loading && (
|
||||
<div className="absolute top-4 right-4 bg-white px-3 py-1 rounded shadow">
|
||||
Loading...
|
||||
|
|
|
|||
|
|
@ -54,34 +54,19 @@ export default function Filters({ filters, onChange }) {
|
|||
/>
|
||||
</div>
|
||||
|
||||
<div className="mt-6 p-3 bg-slate-100 rounded text-xs space-y-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className="w-3 h-3 rounded"
|
||||
style={{ backgroundColor: 'rgb(46, 204, 113)' }}
|
||||
></span>
|
||||
<span>{'< £150k'}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className="w-3 h-3 rounded"
|
||||
style={{ backgroundColor: 'rgb(241, 196, 15)' }}
|
||||
></span>
|
||||
<span>£150k - £300k</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className="w-3 h-3 rounded"
|
||||
style={{ backgroundColor: 'rgb(231, 76, 60)' }}
|
||||
></span>
|
||||
<span>£300k - £500k</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className="w-3 h-3 rounded"
|
||||
style={{ backgroundColor: 'rgb(142, 68, 173)' }}
|
||||
></span>
|
||||
<span>{'> £500k'}</span>
|
||||
<div className="mt-6 p-3 bg-slate-100 rounded text-xs">
|
||||
<div className="mb-2 font-medium">Average Price</div>
|
||||
<div
|
||||
className="h-4 rounded"
|
||||
style={{
|
||||
background: 'linear-gradient(to right, rgb(46, 204, 113), rgb(241, 196, 15), rgb(231, 76, 60), rgb(142, 68, 173))',
|
||||
}}
|
||||
></div>
|
||||
<div className="flex justify-between mt-1">
|
||||
<span>£0</span>
|
||||
<span>£200k</span>
|
||||
<span>£400k</span>
|
||||
<span>£800k+</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import React, { useCallback } from 'react';
|
||||
import React, { useCallback, useRef, useEffect, useState, useMemo } from 'react';
|
||||
import { Map as MapGL } from 'react-map-gl/maplibre';
|
||||
import DeckGL from '@deck.gl/react';
|
||||
import { H3HexagonLayer } from '@deck.gl/geo-layers';
|
||||
|
|
@ -13,32 +13,128 @@ const INITIAL_VIEW = {
|
|||
|
||||
const MAP_STYLE = 'https://basemaps.cartocdn.com/gl/positron-gl-style/style.json';
|
||||
|
||||
// Continuous color scale from green (low) -> yellow -> red -> purple (high)
|
||||
const COLOR_SCALE = [
|
||||
{ price: 0, color: [46, 204, 113] }, // Green
|
||||
{ price: 200000, color: [241, 196, 15] }, // Yellow
|
||||
{ price: 400000, color: [231, 76, 60] }, // Red
|
||||
{ price: 800000, color: [142, 68, 173] }, // Purple
|
||||
];
|
||||
|
||||
function interpolateColor(c1, c2, t) {
|
||||
return [
|
||||
Math.round(c1[0] + (c2[0] - c1[0]) * t),
|
||||
Math.round(c1[1] + (c2[1] - c1[1]) * t),
|
||||
Math.round(c1[2] + (c2[2] - c1[2]) * t),
|
||||
];
|
||||
}
|
||||
|
||||
function priceToColor(price) {
|
||||
if (price < 150000) return [46, 204, 113];
|
||||
if (price < 300000) return [241, 196, 15];
|
||||
if (price < 500000) return [231, 76, 60];
|
||||
return [142, 68, 173];
|
||||
if (price == null || isNaN(price)) return [128, 128, 128]; // Gray for missing data
|
||||
|
||||
// Clamp to scale range
|
||||
if (price <= COLOR_SCALE[0].price) return COLOR_SCALE[0].color;
|
||||
if (price >= COLOR_SCALE[COLOR_SCALE.length - 1].price) {
|
||||
return COLOR_SCALE[COLOR_SCALE.length - 1].color;
|
||||
}
|
||||
|
||||
// Find the two colors to interpolate between
|
||||
for (let i = 0; i < COLOR_SCALE.length - 1; i++) {
|
||||
const lower = COLOR_SCALE[i];
|
||||
const upper = COLOR_SCALE[i + 1];
|
||||
if (price >= lower.price && price <= upper.price) {
|
||||
const t = (price - lower.price) / (upper.price - lower.price);
|
||||
return interpolateColor(lower.color, upper.color, t);
|
||||
}
|
||||
}
|
||||
|
||||
return COLOR_SCALE[COLOR_SCALE.length - 1].color;
|
||||
}
|
||||
|
||||
function zoomToResolution(zoom) {
|
||||
if (zoom < 7) return 6;
|
||||
if (zoom < 8) return 6;
|
||||
if (zoom < 9) return 7;
|
||||
if (zoom < 11) return 8;
|
||||
if (zoom < 13) return 9;
|
||||
if (zoom < 15) return 10;
|
||||
if (zoom < 17) return 11;
|
||||
return 12;
|
||||
if (zoom < 14) return 9;
|
||||
if (zoom < 16) return 10;
|
||||
return 11;
|
||||
}
|
||||
|
||||
export default function Map({ data, onZoom }) {
|
||||
const onViewStateChange = useCallback(
|
||||
({ viewState }) => {
|
||||
onZoom(zoomToResolution(viewState.zoom));
|
||||
},
|
||||
[onZoom]
|
||||
);
|
||||
function getBoundsFromViewState(viewState, width, height) {
|
||||
const { longitude, latitude, zoom } = viewState;
|
||||
|
||||
const layers = [
|
||||
// Clamp latitude to valid Mercator range to avoid math errors
|
||||
const clampedLat = Math.max(-85, Math.min(85, latitude));
|
||||
|
||||
// Web Mercator projection math
|
||||
const TILE_SIZE = 256;
|
||||
const scale = Math.pow(2, zoom);
|
||||
const worldSize = TILE_SIZE * scale;
|
||||
|
||||
// Longitude is linear
|
||||
const degreesPerPixelLng = 360 / worldSize;
|
||||
const halfWidthDeg = (width / 2) * degreesPerPixelLng;
|
||||
|
||||
// Latitude uses Mercator projection (non-linear)
|
||||
// Convert center lat to pixel y, offset by half height, convert back to lat
|
||||
const latRad = clampedLat * Math.PI / 180;
|
||||
const mercatorY = (1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2;
|
||||
const centerPixelY = mercatorY * worldSize;
|
||||
|
||||
const topPixelY = centerPixelY - height / 2;
|
||||
const bottomPixelY = centerPixelY + height / 2;
|
||||
|
||||
// Convert pixel Y back to latitude
|
||||
const pixelYToLat = (pixelY) => {
|
||||
const mercY = Math.max(0.001, Math.min(0.999, pixelY / worldSize)); // Clamp to avoid edge cases
|
||||
const latRadians = Math.atan(Math.sinh(Math.PI * (1 - 2 * mercY)));
|
||||
return latRadians * 180 / Math.PI;
|
||||
};
|
||||
|
||||
const north = Math.min(85, pixelYToLat(topPixelY));
|
||||
const south = Math.max(-85, pixelYToLat(bottomPixelY));
|
||||
const west = Math.max(-180, longitude - halfWidthDeg);
|
||||
const east = Math.min(180, longitude + halfWidthDeg);
|
||||
|
||||
return { south, west, north, east };
|
||||
}
|
||||
|
||||
export default function Map({ data, onViewChange }) {
|
||||
const containerRef = useRef(null);
|
||||
const [viewState, setViewState] = useState(INITIAL_VIEW);
|
||||
const [dimensions, setDimensions] = useState({ width: 0, height: 0 });
|
||||
|
||||
// Track container dimensions with ResizeObserver
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
if (!container) return;
|
||||
|
||||
const observer = new ResizeObserver((entries) => {
|
||||
const { width, height } = entries[0].contentRect;
|
||||
if (width > 0 && height > 0) {
|
||||
setDimensions({ width, height });
|
||||
}
|
||||
});
|
||||
|
||||
observer.observe(container);
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// Notify parent when view or dimensions change
|
||||
useEffect(() => {
|
||||
if (dimensions.width === 0 || dimensions.height === 0) return;
|
||||
|
||||
const bounds = getBoundsFromViewState(viewState, dimensions.width, dimensions.height);
|
||||
const resolution = zoomToResolution(viewState.zoom);
|
||||
|
||||
onViewChange({ resolution, bounds });
|
||||
}, [viewState, dimensions, onViewChange]);
|
||||
|
||||
const handleViewStateChange = useCallback(({ viewState: newViewState }) => {
|
||||
setViewState(newViewState);
|
||||
}, []);
|
||||
|
||||
const layers = useMemo(() => [
|
||||
new H3HexagonLayer({
|
||||
id: 'h3-hexagons',
|
||||
data,
|
||||
|
|
@ -48,15 +144,15 @@ export default function Map({ data, onZoom }) {
|
|||
pickable: true,
|
||||
opacity: 0.7,
|
||||
}),
|
||||
];
|
||||
], [data]);
|
||||
|
||||
return (
|
||||
<div className="flex-1">
|
||||
<div className="flex-1 h-full" ref={containerRef}>
|
||||
<DeckGL
|
||||
initialViewState={INITIAL_VIEW}
|
||||
viewState={viewState}
|
||||
controller
|
||||
layers={layers}
|
||||
onViewStateChange={onViewStateChange}
|
||||
onViewStateChange={handleViewStateChange}
|
||||
>
|
||||
<MapGL mapStyle={MAP_STYLE} />
|
||||
</DeckGL>
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
// Filter configuration constants
|
||||
// Should match backend pipeline/config.py
|
||||
|
||||
export const YEAR_MIN = 1995;
|
||||
export const YEAR_MAX = 2024;
|
||||
export const YEAR_STEP = 1;
|
||||
|
||||
export const PRICE_MIN = 0;
|
||||
export const PRICE_MAX = 2000000;
|
||||
export const PRICE_MAX = 5000000; // £5M max for slider, but no server-side cap
|
||||
export const PRICE_STEP = 50000;
|
||||
|
||||
export const DEFAULT_FILTERS = {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ AGGREGATES_DIR = PROCESSED_DIR / "aggregates"
|
|||
|
||||
# H3 resolutions to generate and serve
|
||||
# https://h3geo.org/docs/core-library/restable/#average-area-in-m2
|
||||
H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11, 12]
|
||||
H3_RESOLUTIONS = [6, 7, 8, 9, 10, 11]
|
||||
DEFAULT_H3_RESOLUTION = 8
|
||||
|
||||
# Year filters
|
||||
|
|
@ -20,4 +20,4 @@ DEFAULT_MAX_YEAR = 2024
|
|||
|
||||
# Price filters
|
||||
DEFAULT_MIN_PRICE = 0
|
||||
DEFAULT_MAX_PRICE = 2_000_000
|
||||
DEFAULT_MAX_PRICE = 100_000_000
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
"""Pipeline CLI to process property data with H3 spatial indexing."""
|
||||
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
from tqdm import tqdm
|
||||
|
||||
from pipeline.sources.postcodes import save_postcodes, DATA_DIR
|
||||
from pipeline.sources.postcodes import save_postcodes
|
||||
from pipeline.sources.property_prices import PropertyPricesSource
|
||||
from pipeline.processors.h3_aggregator import save_aggregates
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ def process_postcodes() -> pl.LazyFrame:
|
|||
df = df.with_columns(
|
||||
pl.struct(["lat", "long"])
|
||||
.map_elements(
|
||||
lambda x: lat_long_to_h3(x["lat"], x["long"], res),
|
||||
# Capture res by value using default argument to avoid closure bug
|
||||
lambda x, res=res: lat_long_to_h3(x["lat"], x["long"], res),
|
||||
return_dtype=pl.Utf8,
|
||||
)
|
||||
.alias(col_name)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ app = FastAPI(title="Property Map API")
|
|||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_credentials=False, # Cannot use True with wildcard origins
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from typing import Any
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi import APIRouter, Query, HTTPException
|
||||
import polars as pl
|
||||
import h3
|
||||
|
||||
|
|
@ -16,19 +16,41 @@ from server.config import (
|
|||
router = APIRouter()
|
||||
|
||||
|
||||
def h3_to_geojson_feature(h3_index: str, properties: dict[str, Any]) -> dict:
|
||||
"""Convert H3 index to GeoJSON feature with polygon geometry."""
|
||||
boundary = h3.cell_to_boundary(h3_index)
|
||||
# h3 returns (lat, lng) pairs, GeoJSON needs [lng, lat]
|
||||
coordinates = [[lng, lat] for lat, lng in boundary]
|
||||
# Close the polygon
|
||||
coordinates.append(coordinates[0])
|
||||
def get_h3_cells_for_bounds(
|
||||
south: float, west: float, north: float, east: float, resolution: int
|
||||
) -> set[str] | None:
|
||||
"""Get all H3 cells that cover a bounding box. Returns None if area too large."""
|
||||
# Clamp to valid ranges
|
||||
south = max(-85, min(85, south))
|
||||
north = max(-85, min(85, north))
|
||||
west = max(-180, min(180, west))
|
||||
east = max(-180, min(180, east))
|
||||
|
||||
return {
|
||||
"type": "Feature",
|
||||
"properties": {"h3": h3_index, **properties},
|
||||
"geometry": {"type": "Polygon", "coordinates": [coordinates]},
|
||||
}
|
||||
# Ensure valid bounds
|
||||
if south >= north or west >= east:
|
||||
return set()
|
||||
|
||||
# If viewport is too large, return None to skip filtering
|
||||
# This prevents H3 from trying to enumerate millions of cells
|
||||
lat_span = north - south
|
||||
lng_span = east - west
|
||||
if lat_span > 20 or lng_span > 30:
|
||||
return None
|
||||
|
||||
# Create polygon from bounds (counter-clockwise winding for H3/GeoJSON)
|
||||
# Order: SW -> NW -> NE -> SE -> SW
|
||||
polygon = [
|
||||
(south, west),
|
||||
(north, west),
|
||||
(north, east),
|
||||
(south, east),
|
||||
(south, west),
|
||||
]
|
||||
|
||||
try:
|
||||
return h3.polygon_to_cells(h3.LatLngPoly(polygon), resolution)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/hexagons")
|
||||
|
|
@ -44,20 +66,41 @@ async def get_hexagons(
|
|||
min_price: float = Query(DEFAULT_MIN_PRICE, description="Minimum average price"),
|
||||
max_price: float = Query(DEFAULT_MAX_PRICE, description="Maximum average price"),
|
||||
bounds: str | None = Query(
|
||||
None, description="Bounding box: lat1,lng1,lat2,lng2"
|
||||
None, description="Bounding box: south,west,north,east"
|
||||
),
|
||||
) -> dict:
|
||||
"""Get aggregated property data as GeoJSON hexagons."""
|
||||
"""Get aggregated property data as GeoJSON hexagons within bounds."""
|
||||
if resolution not in VALID_RESOLUTIONS:
|
||||
resolution = DEFAULT_RESOLUTION
|
||||
|
||||
# Bounds are required for efficient queries
|
||||
if not bounds:
|
||||
raise HTTPException(status_code=400, detail="bounds parameter is required")
|
||||
|
||||
# Parse bounds
|
||||
try:
|
||||
south, west, north, east = map(float, bounds.split(","))
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid bounds format. Use: south,west,north,east"
|
||||
)
|
||||
|
||||
# Load the appropriate resolution file
|
||||
parquet_path = AGGREGATES_DIR / f"res{resolution}.parquet"
|
||||
if not parquet_path.exists():
|
||||
return {"type": "FeatureCollection", "features": []}
|
||||
return {"features": []}
|
||||
|
||||
df = pl.scan_parquet(parquet_path)
|
||||
|
||||
# Get H3 cells that cover the viewport (None if too large to enumerate)
|
||||
viewport_cells = get_h3_cells_for_bounds(south, west, north, east, resolution)
|
||||
|
||||
# Filter to only cells in viewport (skip if viewport too large)
|
||||
if viewport_cells is not None:
|
||||
if len(viewport_cells) == 0:
|
||||
return {"features": []}
|
||||
df = df.filter(pl.col("h3").is_in(viewport_cells))
|
||||
|
||||
# Filter by year range
|
||||
df = df.filter((pl.col("year") >= min_year) & (pl.col("year") <= max_year))
|
||||
|
||||
|
|
@ -80,19 +123,27 @@ async def get_hexagons(
|
|||
(pl.col("avg_price") >= min_price) & (pl.col("avg_price") <= max_price)
|
||||
)
|
||||
|
||||
# Collect and convert to GeoJSON
|
||||
# Limit results to prevent browser crashes
|
||||
MAX_HEXAGONS = 50000
|
||||
df = df.limit(MAX_HEXAGONS)
|
||||
|
||||
# Collect results
|
||||
result = df.collect()
|
||||
|
||||
features = []
|
||||
for row in result.iter_rows(named=True):
|
||||
h3_index = row["h3"]
|
||||
properties = {
|
||||
# Return lightweight response - just h3 index and properties
|
||||
# Frontend H3HexagonLayer will render the geometry
|
||||
# Use to_dicts() which is faster than iter_rows for large results
|
||||
rows = result.to_dicts()
|
||||
features = [
|
||||
{
|
||||
"h3": row["h3"],
|
||||
"count": row["count"],
|
||||
"avg_price": round(row["avg_price"], 2),
|
||||
"median_price": round(row["median_price"], 2) if row["median_price"] else None,
|
||||
"min_price": row["min_price"],
|
||||
"max_price": row["max_price"],
|
||||
}
|
||||
features.append(h3_to_geojson_feature(h3_index, properties))
|
||||
for row in rows
|
||||
]
|
||||
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
return {"features": features, "truncated": len(rows) >= MAX_HEXAGONS}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue