Fix loading times
This commit is contained in:
parent
f685bdca04
commit
2f41c38cc4
4 changed files with 34 additions and 18 deletions
|
|
@ -164,7 +164,8 @@ export default function Map({ data, onViewChange }: MapProps) {
|
||||||
getFillColor: (d) => priceToColor(d.avg_price),
|
getFillColor: (d) => priceToColor(d.avg_price),
|
||||||
extruded: false,
|
extruded: false,
|
||||||
pickable: true,
|
pickable: true,
|
||||||
opacity: 0.7,
|
opacity: 0.5,
|
||||||
|
highPrecision: true,
|
||||||
}),
|
}),
|
||||||
],
|
],
|
||||||
[data]
|
[data]
|
||||||
|
|
|
||||||
|
|
@ -37,5 +37,4 @@ export interface ViewChangeParams {
|
||||||
|
|
||||||
export interface ApiResponse {
|
export interface ApiResponse {
|
||||||
features: HexagonData[];
|
features: HexagonData[];
|
||||||
truncated: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
@ -5,7 +6,16 @@ from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
from server.routes import hexagons
|
from server.routes import hexagons
|
||||||
|
|
||||||
app = FastAPI(title="Property Map API")
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
# Startup: preload all parquet files
|
||||||
|
hexagons.preload_dataframes()
|
||||||
|
yield
|
||||||
|
# Shutdown: nothing to clean up
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(title="Property Map API", lifespan=lifespan)
|
||||||
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
|
import math
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from fastapi import APIRouter, Query, HTTPException
|
from fastapi import APIRouter, Query, HTTPException
|
||||||
import polars as pl
|
import polars as pl
|
||||||
import h3
|
import h3
|
||||||
|
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
from server.config import (
|
from server.config import (
|
||||||
AGGREGATES_DIR,
|
AGGREGATES_DIR,
|
||||||
VALID_RESOLUTIONS,
|
VALID_RESOLUTIONS,
|
||||||
|
|
@ -19,6 +22,12 @@ router = APIRouter()
|
||||||
_df_cache: dict[int, pl.DataFrame] = {}
|
_df_cache: dict[int, pl.DataFrame] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def preload_dataframes() -> None:
|
||||||
|
"""Load all resolution dataframes into cache on startup."""
|
||||||
|
for resolution in tqdm(VALID_RESOLUTIONS, desc="Loading parquet files"):
|
||||||
|
get_cached_df(resolution)
|
||||||
|
|
||||||
|
|
||||||
def get_cached_df(resolution: int) -> pl.DataFrame | None:
|
def get_cached_df(resolution: int) -> pl.DataFrame | None:
|
||||||
"""Get cached dataframe for resolution, loading from disk if needed."""
|
"""Get cached dataframe for resolution, loading from disk if needed."""
|
||||||
if resolution not in _df_cache:
|
if resolution not in _df_cache:
|
||||||
|
|
@ -48,8 +57,8 @@ def query_hexagons_cached(
|
||||||
min_price: int,
|
min_price: int,
|
||||||
max_price: int,
|
max_price: int,
|
||||||
bounds_tuple: tuple[float, float, float, float],
|
bounds_tuple: tuple[float, float, float, float],
|
||||||
) -> tuple[list[dict], bool]:
|
) -> list[dict]:
|
||||||
"""Cached query - returns (features, truncated)."""
|
"""Cached query - returns features list."""
|
||||||
south, west, north, east = bounds_tuple
|
south, west, north, east = bounds_tuple
|
||||||
|
|
||||||
df = get_cached_df(resolution)
|
df = get_cached_df(resolution)
|
||||||
|
|
@ -86,12 +95,6 @@ def query_hexagons_cached(
|
||||||
(pl.col("avg_price") >= min_price) & (pl.col("avg_price") <= max_price)
|
(pl.col("avg_price") >= min_price) & (pl.col("avg_price") <= max_price)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Limit results
|
|
||||||
MAX_HEXAGONS = 50000
|
|
||||||
truncated = len(df) >= MAX_HEXAGONS
|
|
||||||
if truncated:
|
|
||||||
df = df.limit(MAX_HEXAGONS)
|
|
||||||
|
|
||||||
# Build response efficiently using Polars
|
# Build response efficiently using Polars
|
||||||
df = df.select(
|
df = df.select(
|
||||||
[
|
[
|
||||||
|
|
@ -104,7 +107,7 @@ def query_hexagons_cached(
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
return df.to_dicts(), truncated
|
return df.to_dicts()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/hexagons")
|
@router.get("/hexagons")
|
||||||
|
|
@ -136,15 +139,18 @@ async def get_hexagons(
|
||||||
)
|
)
|
||||||
|
|
||||||
# Round bounds to reduce cache misses (0.01 degree ≈ 1km precision)
|
# Round bounds to reduce cache misses (0.01 degree ≈ 1km precision)
|
||||||
|
# Always expand bounds (floor for min, ceil for max) to prevent hexagons
|
||||||
|
# popping in when crossing rounding boundaries
|
||||||
|
precision = 0.01
|
||||||
bounds_tuple = (
|
bounds_tuple = (
|
||||||
round(south, 2),
|
math.floor(south / precision) * precision,
|
||||||
round(west, 2),
|
math.floor(west / precision) * precision,
|
||||||
round(north, 2),
|
math.ceil(north / precision) * precision,
|
||||||
round(east, 2),
|
math.ceil(east / precision) * precision,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Convert prices to int for cache key hashability
|
# Convert prices to int for cache key hashability
|
||||||
features, truncated = query_hexagons_cached(
|
features = query_hexagons_cached(
|
||||||
resolution,
|
resolution,
|
||||||
min_year,
|
min_year,
|
||||||
max_year,
|
max_year,
|
||||||
|
|
@ -153,4 +159,4 @@ async def get_hexagons(
|
||||||
bounds_tuple,
|
bounds_tuple,
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"features": features, "truncated": truncated}
|
return {"features": features}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue