andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Morning improvements

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-17 13:29:03 +00:00
parent 3e9fba5303
commit 53fff3efaa
41 changed files with 2438 additions and 637 deletions
Download patch file Download diff file Expand all files Collapse all files

2
server-rs/src/auth.rs
View file

@ -18,8 +18,6 @@ pub struct PocketBaseUser {
pub id: String,
pub email: String,
#[serde(default)]
pub verified: bool,
#[serde(default)]
pub is_admin: bool,
#[serde(default)]
pub subscription: String,

39
server-rs/src/data/travel_time.rs
View file

@ -236,19 +236,24 @@ impl TravelTimeStore {
}
}
/// Slugify a place name to match travel time file naming convention.
/// "Abbey Hey" → "abbey-hey", "A'Bhuaile Ghlas" → "a-bhuaile-ghlas"
/// Slugify a place name to match Java `originFilename()` convention.
/// Strips non-alphanumeric chars (except spaces/hyphens) first, then collapses
/// whitespace to hyphens. This matches Java's `replaceAll("[^a-z0-9 -]", "")`
/// followed by `replaceAll("\\s+", "-")`.
/// "King's Cross" → "kings-cross", "Abbey Hey" → "abbey-hey"
pub fn slugify(name: &str) -> String {
let mut result = String::with_capacity(name.len());
let mut last_was_hyphen = true; // Start true to skip leading hyphens
for ch in name.chars() {
let lower = ch.to_ascii_lowercase();
if ch.is_ascii_alphanumeric() {
result.push(ch.to_ascii_lowercase());
result.push(lower);
last_was_hyphen = false;
} else if !last_was_hyphen {
} else if (ch == ' ' || ch == '-') && !last_was_hyphen {
result.push('-');
last_was_hyphen = true;
}
// Other non-alphanumeric chars (apostrophes, ampersands, etc.) are stripped
}
if result.ends_with('-') {
result.pop();
@ -266,6 +271,32 @@ mod tests {
assert_eq!(slugify("London"), "london");
}
#[test]
fn slugify_apostrophes_stripped() {
assert_eq!(slugify("King's Cross"), "kings-cross");
assert_eq!(
slugify("Earl's Court tube station"),
"earls-court-tube-station"
);
assert_eq!(slugify("St. Paul's tube station"), "st-pauls-tube-station");
assert_eq!(
slugify("Regent's Park tube station"),
"regents-park-tube-station"
);
}
#[test]
fn slugify_special_chars_stripped() {
assert_eq!(
slugify("Cobham & Stoke d'Abernon railway station"),
"cobham-stoke-dabernon-railway-station"
);
assert_eq!(
slugify("Ravenglass (R&ER) railway station"),
"ravenglass-rer-railway-station"
);
}
#[test]
fn strip_numeric_prefix_basic() {
assert_eq!(

162
server-rs/src/main.rs
View file

@ -35,7 +35,7 @@ use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use state::AppState;
use state::{AppState, SharedState};
#[derive(Parser)]
#[command(
@ -366,19 +366,19 @@ async fn main() -> anyhow::Result<()> {
let token_cache = Arc::new(auth::TokenCache::new());
let state = Arc::new(AppState {
let app_state = AppState {
data: property_data,
grid,
h3_cells,
poi_data,
poi_grid,
place_data,
postcode_data,
poi_data: Arc::new(poi_data),
poi_grid: Arc::new(poi_grid),
place_data: Arc::new(place_data),
postcode_data: Arc::new(postcode_data),
feature_name_to_index,
min_keys,
max_keys,
avg_keys,
poi_category_groups,
poi_category_groups: Arc::new(poi_category_groups),
features_response,
screenshot_url: cli.screenshot_url,
public_url: cli.public_url,
@ -397,14 +397,23 @@ async fn main() -> anyhow::Result<()> {
stripe_secret_key: cli.stripe_secret_key,
stripe_webhook_secret: cli.stripe_webhook_secret,
stripe_referral_coupon_id: cli.stripe_referral_coupon_id,
});
};
let shared = Arc::new(SharedState::new(
app_state,
cli.properties,
cli.postcode_features,
cli.listings_buy,
cli.listings_rent,
));
// Start background PocketBase metrics poller (users, saved searches/properties counts)
pocketbase::start_metrics_poller(state.clone());
pocketbase::start_metrics_poller(shared.clone());
let initial_state = shared.load_state();
let cors = CorsLayer::new()
.allow_origin(
state
initial_state
.public_url
.parse::<axum::http::HeaderValue>()
.expect("public_url must be a valid header value"),
@ -413,183 +422,156 @@ async fn main() -> anyhow::Result<()> {
.allow_headers(AllowHeaders::mirror_request())
.allow_credentials(true);
let state_features = state.clone();
let state_hexagons = state.clone();
let state_postcodes = state.clone();
let state_postcode_lookup = state.clone();
let state_pois = state.clone();
let state_poi_categories = state.clone();
let state_hexagon_properties = state.clone();
let state_hexagon_stats = state.clone();
let state_screenshot = state.clone();
let state_export = state.clone();
let state_crawler = state.clone();
let state_pb = state.clone();
let state_postcode_stats = state.clone();
let state_postcode_properties = state.clone();
let state_places = state.clone();
let state_shorten = state.clone();
let state_short_url = state.clone();
let state_ai_filters = state.clone();
let state_streetview = state.clone();
let state_newsletter = state.clone();
let state_travel_modes = state.clone();
let state_travel_destinations = state.clone();
let state_checkout = state.clone();
let state_stripe_webhook = state.clone();
let state_pricing = state.clone();
let state_invites_list = state.clone();
let state_invites_create = state.clone();
let state_invite_get = state.clone();
let state_redeem_invite = state.clone();
let state_journey = state.clone();
let state_telemetry = state.clone();
// Each route closure captures a clone of `shared` and calls `load_state()`
// at request time to get the latest `Arc<AppState>`. This enables hot-reload:
// the reload endpoint swaps in a new AppState, and subsequent requests pick it up.
macro_rules! s {
() => {
shared.clone()
};
}
let (s1, s2, s3, s4, s5, s6) = (s!(), s!(), s!(), s!(), s!(), s!());
let (s7, s8, s9, s10, s11, s12) = (s!(), s!(), s!(), s!(), s!(), s!());
let (s13, s14, s15, s16, s17, s18) = (s!(), s!(), s!(), s!(), s!(), s!());
let (s19, s20, s21, s22, s23, s24) = (s!(), s!(), s!(), s!(), s!(), s!());
let (s25, s26, s27, s28, s29) = (s!(), s!(), s!(), s!(), s!());
let s_crawler = shared.clone();
let s_pb = shared.clone();
let s_reload = shared.clone();
let api = Router::new()
.route(
"/api/features",
get(move || routes::get_features(state_features.clone())),
get(move || routes::get_features(s1.load_state())),
)
.route(
"/api/hexagons",
get(move |ext, query| routes::get_hexagons(state_hexagons.clone(), ext, query)),
get(move |ext, query| routes::get_hexagons(s2.load_state(), ext, query)),
)
.route(
"/api/postcodes",
get(move |ext, query| routes::get_postcodes(state_postcodes.clone(), ext, query)),
get(move |ext, query| routes::get_postcodes(s3.load_state(), ext, query)),
)
.route(
"/api/postcode/{postcode}",
get(move |path| routes::get_postcode_lookup(state_postcode_lookup.clone(), path)),
get(move |path| routes::get_postcode_lookup(s4.load_state(), path)),
)
.route(
"/api/pois",
get(move |query| routes::get_pois(state_pois.clone(), query)),
get(move |query| routes::get_pois(s5.load_state(), query)),
)
.route(
"/api/poi-categories",
get(move || routes::get_poi_categories(state_poi_categories.clone())),
get(move || routes::get_poi_categories(s6.load_state())),
)
.route(
"/api/places",
get(move |query| routes::get_places(state_places.clone(), query)),
get(move |query| routes::get_places(s7.load_state(), query)),
)
.route(
"/api/travel-modes",
get(move || routes::get_travel_modes(state_travel_modes.clone())),
get(move || routes::get_travel_modes(s8.load_state())),
)
.route(
"/api/travel-destinations",
get(move |query| {
routes::get_travel_destinations(state_travel_destinations.clone(), query)
}),
get(move |query| routes::get_travel_destinations(s9.load_state(), query)),
)
.route(
"/api/journey",
get(move |query| routes::get_journey(state_journey.clone(), query)),
get(move |query| routes::get_journey(s10.load_state(), query)),
)
.route(
"/api/hexagon-properties",
get(move |ext, query| {
routes::get_hexagon_properties(state_hexagon_properties.clone(), ext, query)
}),
get(move |ext, query| routes::get_hexagon_properties(s11.load_state(), ext, query)),
)
.route(
"/api/hexagon-stats",
get(move |ext, query| {
routes::get_hexagon_stats(state_hexagon_stats.clone(), ext, query)
}),
get(move |ext, query| routes::get_hexagon_stats(s12.load_state(), ext, query)),
)
.route(
"/api/postcode-stats",
get(move |ext, query| {
routes::get_postcode_stats(state_postcode_stats.clone(), ext, query)
}),
get(move |ext, query| routes::get_postcode_stats(s13.load_state(), ext, query)),
)
.route(
"/api/postcode-properties",
get(move |ext, query| {
routes::get_postcode_properties(state_postcode_properties.clone(), ext, query)
}),
get(move |ext, query| routes::get_postcode_properties(s14.load_state(), ext, query)),
)
.route(
"/api/screenshot",
get(move |headers, query| {
routes::get_screenshot(state_screenshot.clone(), headers, query)
}),
get(move |headers, query| routes::get_screenshot(s15.load_state(), headers, query)),
)
.route(
"/api/export",
get(move |headers, ext, query| {
routes::get_export(state_export.clone(), headers, ext, query)
routes::get_export(s16.load_state(), headers, ext, query)
})
.layer(ConcurrencyLimitLayer::new(3)),
)
.route("/api/me", get(routes::get_me))
.route(
"/api/shorten",
post(move |body| routes::post_shorten(state_shorten.clone(), body)),
post(move |body| routes::post_shorten(s17.load_state(), body)),
)
.route(
"/api/ai-filters",
post(move |ext, body| routes::post_ai_filters(state_ai_filters.clone(), ext, body))
post(move |ext, body| routes::post_ai_filters(s18.load_state(), ext, body))
.layer(ConcurrencyLimitLayer::new(5)),
)
.route(
"/api/streetview",
get(move |query| routes::get_streetview(state_streetview.clone(), query)),
get(move |query| routes::get_streetview(s19.load_state(), query)),
)
.route(
"/api/newsletter",
patch(move |ext, body| routes::patch_newsletter(state_newsletter.clone(), ext, body)),
patch(move |ext, body| routes::patch_newsletter(s20.load_state(), ext, body)),
)
.route(
"/api/pricing",
get(move || routes::get_pricing(state_pricing.clone())),
get(move || routes::get_pricing(s21.load_state())),
)
.route(
"/api/checkout",
post(move |ext, body| routes::post_checkout(state_checkout.clone(), ext, body))
post(move |ext, body| routes::post_checkout(s22.load_state(), ext, body))
.layer(ConcurrencyLimitLayer::new(10)),
)
.route(
"/api/stripe-webhook",
post(move |headers, body| {
routes::post_stripe_webhook(state_stripe_webhook.clone(), headers, body)
}),
post(move |headers, body| routes::post_stripe_webhook(s23.load_state(), headers, body)),
)
.route(
"/api/invites",
get(move |ext| routes::get_invites(state_invites_list.clone(), ext)).post(
move |ext, body| routes::post_invites(state_invites_create.clone(), ext, body),
),
get(move |ext| routes::get_invites(s24.load_state(), ext))
.post(move |ext, body| routes::post_invites(s25.load_state(), ext, body)),
)
.route(
"/api/invite/{code}",
get(move |ext, path| routes::get_invite(state_invite_get.clone(), ext, path)),
get(move |ext, path| routes::get_invite(s26.load_state(), ext, path)),
)
.route(
"/api/redeem-invite",
post(move |ext, body| {
routes::post_redeem_invite(state_redeem_invite.clone(), ext, body)
}),
post(move |ext, body| routes::post_redeem_invite(s27.load_state(), ext, body)),
)
.route(
"/s/{code}",
get(move |path| routes::get_short_url(state_short_url.clone(), path)),
get(move |path| routes::get_short_url(s28.load_state(), path)),
)
.route(
"/api/telemetry",
post(move |ext, headers, body| {
let _ = state_telemetry.clone();
let _ = s29.load_state();
routes::post_telemetry(ext, headers, body)
}),
)
.route(
"/api/reload",
post(move || routes::post_reload(s_reload.clone())),
);
// Add tile routes
let reader_tile = tile_reader.clone();
let reader_style = tile_reader.clone();
let public_url_tiles = state.public_url.clone();
let public_url_tiles = initial_state.public_url.clone();
let api = api
.route(
"/api/tiles/{z}/{x}/{y}",
@ -609,7 +591,7 @@ async fn main() -> anyhow::Result<()> {
)
.route(
"/pb/{*rest}",
any(move |req| routes::proxy_to_pocketbase(state_pb.clone(), req)),
any(move |req| routes::proxy_to_pocketbase(s_pb.load_state(), req)),
);
let app = if let Some(ref dist) = cli.dist {
@ -621,7 +603,7 @@ async fn main() -> anyhow::Result<()> {
.layer(middleware::from_fn(auth::auth_middleware))
.layer(middleware::from_fn(
move |req: axum::extract::Request, next: middleware::Next| {
let st = state_crawler.clone();
let st = s_crawler.load_state();
async move {
// Inject state into request extensions for auth + OG middleware
let (mut parts, body) = req.into_parts();

37
server-rs/src/metrics.rs
View file

@ -4,12 +4,47 @@ use axum::http::StatusCode;
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use metrics::{counter, gauge, histogram};
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
use std::time::Instant;
/// Initialize the Prometheus metrics exporter and return a handle for rendering metrics.
///
/// Configures histogram bucket boundaries so the exporter renders Prometheus histograms
/// (with `_bucket` suffix) instead of summaries. Without this, `histogram_quantile()`
/// queries in Grafana find no `_bucket` metrics and return empty.
pub fn init_metrics() -> PrometheusHandle {
// Standard Prometheus buckets for HTTP latencies (seconds)
const LATENCY_BUCKETS: &[f64] = &[
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
// Wider buckets for screenshot generation (can take 30s+)
const SCREENSHOT_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0];
// Count-based buckets for response sizes (number of hexagons/postcodes returned)
const RESPONSE_COUNT_BUCKETS: &[f64] = &[
10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0,
];
PrometheusBuilder::new()
.set_buckets_for_metric(
Matcher::Full("http_request_duration_seconds".to_string()),
LATENCY_BUCKETS,
)
.expect("Failed to set HTTP latency buckets")
.set_buckets_for_metric(
Matcher::Full("screenshot_duration_seconds".to_string()),
SCREENSHOT_BUCKETS,
)
.expect("Failed to set screenshot duration buckets")
.set_buckets_for_metric(
Matcher::Full("hexagons_response_count".to_string()),
RESPONSE_COUNT_BUCKETS,
)
.expect("Failed to set hexagons response count buckets")
.set_buckets_for_metric(
Matcher::Full("postcodes_response_count".to_string()),
RESPONSE_COUNT_BUCKETS,
)
.expect("Failed to set postcodes response count buckets")
.install_recorder()
.expect("Failed to install Prometheus recorder")
}

7
server-rs/src/parsing/fields.rs
View file

@ -4,8 +4,9 @@ use axum::http::StatusCode;
use rustc_hash::FxHashMap;
/// Parse an optional `?fields=` query param into feature indices for selective aggregation.
/// Returns `None` if fields is `None` (all features included), or `Some(indices)` if specified.
/// Returns an error if any field name is unknown.
/// Returns `None` if fields param is absent (all features included).
/// Returns `Some(vec![])` if fields is present but empty (no features — count only).
/// Returns `Some(indices)` for named fields. Errors on unknown field names.
pub fn parse_field_indices(
fields: Option<&str>,
name_to_index: &FxHashMap<String, usize>,
@ -14,7 +15,7 @@ pub fn parse_field_indices(
return Ok(None);
};
if fields_str.is_empty() {
return Ok(None);
return Ok(Some(vec![]));
}
let mut indices = Vec::new();
for name in fields_str.split(',') {

5
server-rs/src/pocketbase.rs
View file

@ -763,11 +763,12 @@ pub async fn ensure_oauth_providers(
/// Spawn a background task that polls PocketBase every 60 seconds for collection counts
/// and exposes them as Prometheus gauges.
pub fn start_metrics_poller(state: Arc<AppState>) {
pub fn start_metrics_poller(shared: Arc<crate::state::SharedState>) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
let state = shared.load_state();
poll_pocketbase_counts(&state).await;
}
});
@ -815,7 +816,7 @@ async fn poll_pocketbase_counts(state: &AppState) {
("type", "referral"),
),
(
Some(r##"used_by_id!=""##),
Some(r#"used_by_id!="""#),
"invites_total",
("type", "redeemed"),
),

2
server-rs/src/routes.rs
View file

@ -16,6 +16,7 @@ mod postcode_stats;
mod postcodes;
pub(crate) mod pricing;
pub(crate) mod properties;
mod reload;
mod screenshot;
mod shorten;
mod stats;
@ -45,6 +46,7 @@ pub use postcode_stats::get_postcode_stats;
pub use postcodes::{get_postcode_lookup, get_postcodes};
pub use pricing::get_pricing;
pub use properties::get_hexagon_properties;
pub use reload::post_reload;
pub use screenshot::{fetch_screenshot_bytes, get_screenshot};
pub use shorten::{get_short_url, post_shorten};
pub use streetview::get_streetview;

8
server-rs/src/routes/ai_filters.rs
View file

@ -510,14 +510,6 @@ pub async fn post_ai_filters(
.0
.ok_or((StatusCode::UNAUTHORIZED, "Login required".into()))?;
// Email verification check (skipped in dev mode)
if !user.verified && !state.is_dev {
return Err((
StatusCode::FORBIDDEN,
"Please verify your email to use AI filters".into(),
));
}
// Check weekly token usage
let current_week = current_week_number();
let (stored_tokens, stored_week) = fetch_ai_usage(&state, &user.id).await?;

330
server-rs/src/routes/hexagons.rs
View file

@ -17,8 +17,8 @@ use crate::consts::{DEMO_BOUNDS, MAX_CELLS_PER_REQUEST};
use crate::data::travel_time::TravelData;
use crate::licensing::check_license_bounds;
use crate::parsing::{
bounds_intersect, cell_for_row_cached, h3_cell_bounds, needs_parent, parse_field_indices,
parse_filters, require_bounds, row_passes_filters, validate_h3_resolution,
cell_for_row_cached, needs_parent, parse_field_indices, parse_filters, require_bounds,
row_passes_filters, validate_h3_resolution,
};
use crate::routes::travel_time::{parse_optional_travel, TravelTimeAgg};
use crate::state::AppState;
@ -26,6 +26,28 @@ use crate::state::AppState;
/// Row count threshold above which we use rayon parallel aggregation.
const PARALLEL_THRESHOLD: usize = 50_000;
/// Per-thread aggregation result: feature accumulators + travel time accumulators.
type ChunkResult = (
FxHashMap<u64, Aggregator>,
Vec<FxHashMap<u64, TravelTimeAgg>>,
);
/// Maximum center-to-vertex distance in degrees per H3 resolution.
/// Generous for UK latitudes (49°–61°) so we never false-exclude a visible cell.
/// Used for cheap center-based bounds filtering instead of computing full cell boundary.
const H3_CENTER_BUFFERS: [f64; 13] = [
5.0, 2.0, 1.0, 0.5, // res 0–3 (unused in practice)
0.50, // res 4
0.20, // res 5
0.08, // res 6
0.03, // res 7
0.012, // res 8
0.005, // res 9
0.002, // res 10
0.001, // res 11
0.0005, // res 12
];
#[derive(Serialize)]
pub struct HexagonsResponse {
features: Vec<Map<String, Value>>,
@ -45,7 +67,10 @@ pub struct HexagonParams {
travel: Option<String>,
}
/// Build feature maps from aggregated cell data, filtering to only cells that intersect the query bounds.
/// Build feature maps from aggregated cell data, filtering to only cells whose
/// center is within the query bounds (expanded by a resolution-dependent buffer).
/// This is much cheaper than the previous approach of computing full cell boundaries
/// (6 vertices per cell) — just 4 float comparisons per cell.
#[allow(clippy::too_many_arguments)]
fn build_feature_maps(
groups: &FxHashMap<u64, Aggregator>,
@ -55,44 +80,69 @@ fn build_feature_maps(
num_features: usize,
indices: Option<&[usize]>,
query_bounds: (f64, f64, f64, f64),
resolution: h3o::Resolution,
travel_aggs: &[FxHashMap<u64, TravelTimeAgg>],
travel_field_keys: &[String],
) -> Vec<Map<String, Value>> {
let mut features = Vec::with_capacity(groups.len());
let (q_south, q_west, q_north, q_east) = query_bounds;
// Expand bounds by resolution-dependent buffer for center-based filtering
let buf = H3_CENTER_BUFFERS[resolution as usize];
let bound_south = q_south - buf;
let bound_north = q_north + buf;
let bound_west = q_west - buf;
let bound_east = q_east + buf;
// Pre-compute travel time key strings (avoids per-cell format!())
let travel_keys: Vec<(String, String, String)> = travel_field_keys
.iter()
.map(|key| {
(
format!("min_{key}"),
format!("max_{key}"),
format!("avg_{key}"),
)
})
.collect();
// Pre-compute default feature indices to avoid per-cell Box<dyn Iterator> allocation
let default_indices: Vec<usize>;
let feat_indices: &[usize] = match indices {
Some(idx) => idx,
None => {
default_indices = (0..num_features).collect();
&default_indices
}
};
for (&cell_id, aggregation) in groups {
let Some(cell) = h3o::CellIndex::try_from(cell_id).ok() else {
continue;
};
// Filter out cells that don't intersect the query bounds
let (c_south, c_west, c_north, c_east) = h3_cell_bounds(cell, 0.0);
if !bounds_intersect(
c_south, c_west, c_north, c_east, q_south, q_west, q_north, q_east,
) {
// Center is already needed for lat/lon output — reuse for bounds check
let center: h3o::LatLng = cell.into();
let lat = center.lat();
let lng = center.lng();
// Center-based bounds check: 4 comparisons instead of computing 6 boundary vertices
if lat < bound_south || lat > bound_north || lng < bound_west || lng > bound_east {
continue;
}
let mut map = Map::new();
map.insert("h3".into(), Value::String(cell.to_string()));
map.insert("count".into(), Value::Number(aggregation.count.into()));
let center: h3o::LatLng = cell.into();
if let (Some(lat), Some(lon)) = (
serde_json::Number::from_f64(center.lat()),
serde_json::Number::from_f64(center.lng()),
if let (Some(lat_num), Some(lon_num)) = (
serde_json::Number::from_f64(lat),
serde_json::Number::from_f64(lng),
) {
map.insert("lat".into(), Value::Number(lat));
map.insert("lon".into(), Value::Number(lon));
map.insert("lat".into(), Value::Number(lat_num));
map.insert("lon".into(), Value::Number(lon_num));
}
let iter: Box<dyn Iterator<Item = usize>> = if let Some(idx) = indices {
Box::new(idx.iter().copied())
} else {
Box::new(0..num_features)
};
for feat_index in iter {
for &feat_index in feat_indices {
if aggregation.feat_counts[feat_index] > 0 {
let avg = aggregation.sums[feat_index] / aggregation.feat_counts[feat_index] as f64;
if let (Some(min_num), Some(max_num), Some(avg_num)) = (
@ -107,20 +157,19 @@ fn build_feature_maps(
}
}
// Add travel time aggregation fields
// Add travel time aggregation fields (using pre-computed key strings)
for (ti, agg_map) in travel_aggs.iter().enumerate() {
if let Some(agg) = agg_map.get(&cell_id) {
if agg.count > 0 {
let key = &travel_field_keys[ti];
let avg = agg.sum / agg.count as f64;
if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) {
map.insert(format!("min_{key}"), Value::Number(nm));
map.insert(travel_keys[ti].0.clone(), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) {
map.insert(format!("max_{key}"), Value::Number(nm));
map.insert(travel_keys[ti].1.clone(), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(avg) {
map.insert(format!("avg_{key}"), Value::Number(nm));
map.insert(travel_keys[ti].2.clone(), Value::Number(nm));
}
}
}
@ -207,19 +256,31 @@ pub async fn get_hexagons(
.map(|_| FxHashMap::default())
.collect();
// Collect row indices for threshold-based sequential/parallel aggregation
let row_indices = state.grid.query(south, west, north, east);
// O(grid cells) count — no allocation. Used for parallel threshold decision.
let row_count = state.grid.count_in_bounds(south, west, north, east);
let t_grid = t0.elapsed();
if row_indices.len() >= PARALLEL_THRESHOLD && !has_travel {
// Parallel path: split rows across rayon threads, each with local accumulators
let parallel = row_count >= PARALLEL_THRESHOLD;
if parallel {
// Parallel: collect row indices for par_chunks, split across rayon threads.
// Now handles travel time too (postcode interner & travel data are thread-safe).
let row_indices = state.grid.query(south, west, north, east);
let chunk_size = (row_indices.len() / rayon::current_num_threads()).max(1000);
let thread_results: Vec<FxHashMap<u64, Aggregator>> = row_indices
let thread_results: Vec<ChunkResult> = row_indices
.par_chunks(chunk_size)
.map(|chunk| {
let mut local_groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
let mut local_travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> = (0
..travel_entries.len())
.map(|_| FxHashMap::default())
.collect();
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
for &row_idx in chunk {
let mut travel_minutes: Vec<Option<i16>> =
Vec::with_capacity(travel_entries.len());
'row: for &row_idx in chunk {
let row = row_idx as usize;
if !row_passes_filters(
row,
@ -230,6 +291,32 @@ pub async fn get_hexagons(
) {
continue;
}
if has_travel {
travel_minutes.clear();
let postcode = pc_interner.resolve(&pc_keys[row]);
for (ti, entry) in travel_entries.iter().enumerate() {
let row_data = travel_data[ti].get(postcode);
let minutes = row_data.map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
travel_minutes.push(minutes);
if let (Some(fmin), Some(fmax)) =
(entry.filter_min, entry.filter_max)
{
match minutes {
Some(mins)
if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => continue 'row,
}
}
}
}
let cell_id = cell_for_row_cached(
row,
precomputed,
@ -237,6 +324,7 @@ pub async fn get_hexagons(
need_parent,
&mut h3_cache,
);
let agg = local_groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
@ -251,91 +339,108 @@ pub async fn get_hexagons(
} else {
agg.add_row(feature_data, row, num_features, &quant);
}
}
local_groups
})
.collect();
// Merge thread-local results into the main groups map
for local_groups in thread_results {
for (cell_id, local_agg) in local_groups {
let agg = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
agg.merge(&local_agg);
}
}
} else {
// Sequential path (also handles travel time which needs postcode lookups)
let mut travel_minutes: Vec<Option<i16>> = Vec::with_capacity(travel_entries.len());
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
'row: for &row_idx in &row_indices {
let row = row_idx as usize;
// Regular filters
if !row_passes_filters(
row,
&parsed_filters,
&parsed_enum_filters,
feature_data,
num_features,
) {
continue;
}
// Travel time filter: check each entry with a range
if has_travel {
travel_minutes.clear();
let postcode = pc_interner.resolve(&pc_keys[row]);
for (ti, entry) in travel_entries.iter().enumerate() {
let row_data = travel_data[ti].get(postcode);
let minutes = row_data.map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
travel_minutes.push(minutes);
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
match minutes {
Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => continue 'row, // Filtered out (jump to next row_idx)
for (ti, minutes) in travel_minutes.iter().enumerate() {
if let Some(mins) = minutes {
let tagg = local_travel_aggs[ti]
.entry(cell_id)
.or_insert_with(TravelTimeAgg::new);
tagg.add(*mins as f32);
}
}
}
(local_groups, local_travel_aggs)
})
.collect();
// Merge thread-local results into the main accumulators
for (local_groups, local_travel) in thread_results {
for (cell_id, local_agg) in local_groups {
groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features))
.merge(&local_agg);
}
let cell_id =
cell_for_row_cached(row, precomputed, h3_res, need_parent, &mut h3_cache);
// Aggregate regular features
let aggregation = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
if let Some(sel_indices) = field_indices.as_deref() {
aggregation.add_row_selective(
feature_data,
row,
num_features,
sel_indices,
&quant,
);
} else {
aggregation.add_row(feature_data, row, num_features, &quant);
}
// Aggregate travel time
for (ti, minutes) in travel_minutes.iter().enumerate() {
if let Some(mins) = minutes {
let agg = travel_aggs[ti]
for (ti, local_ta) in local_travel.into_iter().enumerate() {
for (cell_id, local_tt) in local_ta {
travel_aggs[ti]
.entry(cell_id)
.or_insert_with(TravelTimeAgg::new);
agg.add(*mins as f32);
.or_insert_with(TravelTimeAgg::new)
.merge(&local_tt);
}
}
}
} else {
// Sequential: use for_each_in_bounds to avoid Vec<u32> allocation
let mut travel_minutes: Vec<Option<i16>> = Vec::with_capacity(travel_entries.len());
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
state
.grid
.for_each_in_bounds(south, west, north, east, |row_idx| {
let row = row_idx as usize;
if !row_passes_filters(
row,
&parsed_filters,
&parsed_enum_filters,
feature_data,
num_features,
) {
return;
}
if has_travel {
travel_minutes.clear();
let postcode = pc_interner.resolve(&pc_keys[row]);
for (ti, entry) in travel_entries.iter().enumerate() {
let row_data = travel_data[ti].get(postcode);
let minutes = row_data.map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
travel_minutes.push(minutes);
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
match minutes {
Some(mins)
if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => return,
}
}
}
}
let cell_id =
cell_for_row_cached(row, precomputed, h3_res, need_parent, &mut h3_cache);
let aggregation = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
if let Some(sel_indices) = field_indices.as_deref() {
aggregation.add_row_selective(
feature_data,
row,
num_features,
sel_indices,
&quant,
);
} else {
aggregation.add_row(feature_data, row, num_features, &quant);
}
for (ti, minutes) in travel_minutes.iter().enumerate() {
if let Some(mins) = minutes {
let agg = travel_aggs[ti]
.entry(cell_id)
.or_insert_with(TravelTimeAgg::new);
agg.add(*mins as f32);
}
}
});
};
let t_agg = t0.elapsed();
@ -348,6 +453,7 @@ pub async fn get_hexagons(
num_features,
field_indices.as_deref(),
(south, west, north, east),
h3_res,
&travel_aggs,
&travel_field_keys,
);
@ -357,11 +463,10 @@ pub async fn get_hexagons(
features.truncate(MAX_CELLS_PER_REQUEST);
}
let parallel = row_indices.len() >= PARALLEL_THRESHOLD && !has_travel;
let t_total = t0.elapsed();
info!(
resolution,
rows = row_indices.len(),
rows = row_count,
parallel,
cells_before_filter = groups.len(),
cells_after_filter = features.len(),
@ -369,8 +474,11 @@ pub async fn get_hexagons(
bounds = format_args!("{:.4},{:.4},{:.4},{:.4}", south, west, north, east),
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
fields = field_indices.as_ref().map(|v| v.len() as i32).unwrap_or(-1),
travel_entries = travel_entries.len(),
agg_ms = format_args!("{:.1}", t_agg.as_secs_f64() * 1000.0),
grid_ms = format_args!("{:.1}", t_grid.as_secs_f64() * 1000.0),
agg_ms = format_args!("{:.1}", (t_agg - t_grid).as_secs_f64() * 1000.0),
json_ms = format_args!("{:.1}", (t_total - t_agg).as_secs_f64() * 1000.0),
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
"GET /api/hexagons"
);

2
server-rs/src/routes/pois.rs
View file

@ -128,7 +128,7 @@ pub struct POICategoriesResponse {
}
pub async fn get_poi_categories(state: Arc<AppState>) -> Json<POICategoriesResponse> {
let groups: Vec<POICategoryGroup> = state.poi_category_groups.clone();
let groups: Vec<POICategoryGroup> = state.poi_category_groups.to_vec();
let total: usize = groups.iter().map(|group| group.categories.len()).sum();
info!(

5
server-rs/src/routes/postcodes.rs
View file

@ -177,6 +177,8 @@ pub async fn get_postcodes(
}
}
let t_agg = t0.elapsed();
// Build response, filtering postcodes to only those whose polygon intersects query bounds
let mut features = Vec::with_capacity(postcode_aggs.len());
let postcodes_before_filter = postcode_aggs.len();
@ -288,7 +290,10 @@ pub async fn get_postcodes(
bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east),
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
fields = field_indices.as_ref().map(|v| v.len() as i32).unwrap_or(-1),
travel_entries = travel_entries.len(),
agg_ms = format_args!("{:.1}", t_agg.as_secs_f64() * 1000.0),
json_ms = format_args!("{:.1}", (t_total - t_agg).as_secs_f64() * 1000.0),
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
"GET /api/postcodes"
);

179
server-rs/src/routes/reload.rs Normal file
View file

@ -0,0 +1,179 @@
use std::sync::Arc;
use std::time::Instant;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Json, Response};
use serde_json::json;
use tracing::{info, warn};
use crate::consts::GRID_CELL_SIZE;
use crate::data::{self, PropertyData};
use crate::metrics::record_data_stats;
use crate::routes::{build_features_response, build_system_prompt};
use crate::state::{AppState, SharedState};
use crate::utils::GridIndex;
pub async fn post_reload(shared: Arc<SharedState>) -> Response {
if !shared.try_start_reload() {
return (StatusCode::CONFLICT, "Reload already in progress").into_response();
}
info!("Reload triggered — rebuilding property data");
let start = Instant::now();
// shared is cloned so we retain a reference after spawn_blocking
let sh = Arc::clone(&shared);
let result = tokio::task::spawn_blocking(move || rebuild_data(&sh, start)).await;
// Always clear the reload flag
shared.finish_reload();
match result {
Ok(Ok((rows, features, elapsed_ms))) => Json(json!({
"status": "ok",
"rows": rows,
"features": features,
"elapsed_ms": elapsed_ms,
}))
.into_response(),
Ok(Err(err)) => {
warn!("Reload failed: {err:#}");
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({ "error": format!("{err:#}") })),
)
.into_response()
}
Err(err) => {
warn!("Reload task panicked: {err}");
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(json!({ "error": format!("Reload task panicked: {err}") })),
)
.into_response()
}
}
}
fn rebuild_data(shared: &SharedState, start: Instant) -> anyhow::Result<(usize, usize, u128)> {
let old = shared.load_state();
// 1. Load PropertyData from parquet files
let property_data = PropertyData::load(
&shared.properties_path,
&shared.postcode_features_path,
&shared.listings_buy_path,
&shared.listings_rent_path,
)?;
let row_count = property_data.lat.len();
let feature_count = property_data.num_features;
// 2. Build spatial grid index
info!("Reload: building spatial grid index");
let grid = GridIndex::build(&property_data.lat, &property_data.lon, GRID_CELL_SIZE);
// 3. Precompute H3 cells
info!("Reload: precomputing H3 cells");
let h3_cells = data::precompute_h3(&property_data.lat, &property_data.lon)?;
// 4. Build feature lookup tables
let feature_name_to_index = property_data
.feature_names
.iter()
.enumerate()
.map(|(idx, name)| (name.clone(), idx))
.collect();
let min_keys = property_data
.feature_names
.iter()
.map(|n| format!("min_{n}"))
.collect();
let max_keys = property_data
.feature_names
.iter()
.map(|n| format!("max_{n}"))
.collect();
let avg_keys = property_data
.feature_names
.iter()
.map(|n| format!("avg_{n}"))
.collect();
// 5. Build features response and AI prompt
let features_response = build_features_response(&property_data);
let mode_destinations: Vec<(String, usize)> = old
.travel_time_store
.available_modes
.iter()
.map(|mode| {
let count = old
.travel_time_store
.destinations
.get(mode.as_str())
.map(|slugs| slugs.len())
.unwrap_or(0);
(mode.clone(), count)
})
.filter(|(_, count)| *count > 0)
.collect();
let ai_filters_system_prompt = build_system_prompt(&features_response, &mode_destinations);
// 6. Update data metrics
record_data_stats(
row_count,
old.poi_data.lat.len(),
old.postcode_data.postcodes.len(),
);
// 7. Build new AppState, sharing unchanged fields via Arc
let new_state = AppState {
data: property_data,
grid,
h3_cells,
feature_name_to_index,
min_keys,
max_keys,
avg_keys,
features_response,
ai_filters_system_prompt,
// Shared across reloads (Arc clone is cheap)
poi_data: Arc::clone(&old.poi_data),
poi_grid: Arc::clone(&old.poi_grid),
place_data: Arc::clone(&old.place_data),
postcode_data: Arc::clone(&old.postcode_data),
poi_category_groups: Arc::clone(&old.poi_category_groups),
travel_time_store: Arc::clone(&old.travel_time_store),
token_cache: Arc::clone(&old.token_cache),
// Config (cheap clone)
screenshot_url: old.screenshot_url.clone(),
public_url: old.public_url.clone(),
is_dev: old.is_dev,
index_html: old.index_html.clone(),
http_client: old.http_client.clone(),
pocketbase_url: old.pocketbase_url.clone(),
pocketbase_admin_email: old.pocketbase_admin_email.clone(),
pocketbase_admin_password: old.pocketbase_admin_password.clone(),
gemini_api_key: old.gemini_api_key.clone(),
gemini_model: old.gemini_model.clone(),
google_maps_api_key: old.google_maps_api_key.clone(),
stripe_secret_key: old.stripe_secret_key.clone(),
stripe_webhook_secret: old.stripe_webhook_secret.clone(),
stripe_referral_coupon_id: old.stripe_referral_coupon_id.clone(),
};
// 8. Atomic swap
shared.swap_state(new_state);
let elapsed = start.elapsed();
info!(
rows = row_count,
features = feature_count,
elapsed_ms = elapsed.as_millis(),
"Reload complete"
);
Ok((row_count, feature_count, elapsed.as_millis()))
}

14
server-rs/src/routes/travel_time.rs
View file

@ -93,4 +93,18 @@ impl TravelTimeAgg {
self.sum += value as f64;
self.count += 1;
}
/// Merge another aggregator's results into this one (for parallel reduction).
pub fn merge(&mut self, other: &TravelTimeAgg) {
if other.count > 0 {
if other.min < self.min {
self.min = other.min;
}
if other.max > self.max {
self.max = other.max;
}
self.sum += other.sum;
self.count += other.count;
}
}
}

88
server-rs/src/state.rs
View file

@ -1,5 +1,8 @@
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use parking_lot::RwLock;
use rustc_hash::FxHashMap;
use crate::auth::TokenCache;
@ -10,16 +13,12 @@ use crate::routes::FeaturesResponse;
use crate::utils::GridIndex;
pub struct AppState {
// --- Rebuilt on reload ---
pub data: PropertyData,
pub grid: GridIndex,
/// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (12).
/// Parent cells for lower resolutions derived via CellIndex::parent().
pub h3_cells: Vec<u64>,
pub poi_data: POIData,
pub poi_grid: GridIndex,
pub place_data: PlaceData,
/// Postcode boundary data for high-zoom rendering
pub postcode_data: PostcodeData,
/// O(1) lookup: feature name → index in feature_names/feature_data
pub feature_name_to_index: FxHashMap<String, usize>,
/// Precomputed JSON key names: "min_{feature_name}" for each feature
@ -28,10 +27,25 @@ pub struct AppState {
pub max_keys: Vec<String>,
/// Precomputed JSON key names: "avg_{feature_name}" for each feature
pub avg_keys: Vec<String>,
/// Precomputed POI category groups (sorted)
pub poi_category_groups: Vec<POICategoryGroup>,
/// Precomputed features response for /api/features endpoint
pub features_response: FeaturesResponse,
/// Complete system prompt for AI filters (features + examples + instructions)
pub ai_filters_system_prompt: String,
// --- Shared across reloads (Arc for cheap cloning) ---
pub poi_data: Arc<POIData>,
pub poi_grid: Arc<GridIndex>,
pub place_data: Arc<PlaceData>,
/// Postcode boundary data for high-zoom rendering
pub postcode_data: Arc<PostcodeData>,
/// Precomputed POI category groups (sorted)
pub poi_category_groups: Arc<Vec<POICategoryGroup>>,
/// Precomputed travel time data store
pub travel_time_store: Arc<TravelTimeStore>,
/// Token validation cache (60s TTL)
pub token_cache: Arc<TokenCache>,
// --- Config (cheap to clone) ---
/// URL of the screenshot service (e.g. http://screenshot:8002)
pub screenshot_url: String,
/// Public-facing URL for absolute og:image URLs (e.g. https://perfectpostcodes.dev)
@ -52,12 +66,6 @@ pub struct AppState {
pub gemini_api_key: String,
/// Gemini model name (e.g. gemini-2.0-flash)
pub gemini_model: String,
/// Precomputed travel time data store
pub travel_time_store: Arc<TravelTimeStore>,
/// Token validation cache (60s TTL)
pub token_cache: Arc<TokenCache>,
/// Complete system prompt for AI filters (features + examples + instructions)
pub ai_filters_system_prompt: String,
/// Google Maps API key for Street View metadata lookups
pub google_maps_api_key: String,
/// Stripe secret key for creating checkout sessions
@ -67,3 +75,57 @@ pub struct AppState {
/// Stripe Coupon ID for referral discounts
pub stripe_referral_coupon_id: String,
}
/// Wraps AppState with atomic swap capability for hot-reloading.
/// Route handlers call `load_state()` to get the current snapshot.
/// The reload endpoint builds a new AppState and swaps it in atomically.
pub struct SharedState {
current: RwLock<Arc<AppState>>,
reloading: AtomicBool,
/// Paths needed for data reload
pub properties_path: PathBuf,
pub postcode_features_path: PathBuf,
pub listings_buy_path: PathBuf,
pub listings_rent_path: PathBuf,
}
impl SharedState {
pub fn new(
state: AppState,
properties_path: PathBuf,
postcode_features_path: PathBuf,
listings_buy_path: PathBuf,
listings_rent_path: PathBuf,
) -> Self {
Self {
current: RwLock::new(Arc::new(state)),
reloading: AtomicBool::new(false),
properties_path,
postcode_features_path,
listings_buy_path,
listings_rent_path,
}
}
/// Get the current AppState snapshot. Cheap (Arc clone under a brief read lock).
pub fn load_state(&self) -> Arc<AppState> {
self.current.read().clone()
}
/// Atomically swap in a new AppState. Old state is dropped when all references are gone.
pub fn swap_state(&self, new_state: AppState) {
*self.current.write() = Arc::new(new_state);
}
/// Try to mark reload as in-progress. Returns false if already reloading.
pub fn try_start_reload(&self) -> bool {
self.reloading
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
}
/// Mark reload as complete.
pub fn finish_reload(&self) {
self.reloading.store(false, Ordering::Release);
}
}

43
server-rs/src/utils/grid_index.rs
View file

@ -138,6 +138,26 @@ impl GridIndex {
result
}
/// Count the number of row indices within the given bounds without allocating.
/// O(grid cells in bounds) — much cheaper than query() for threshold decisions.
pub fn count_in_bounds(&self, south: f64, west: f64, north: f64, east: f64) -> usize {
let Some((row_min, row_max, col_min, col_max)) =
self.clamp_bounds(south, west, north, east)
else {
return 0;
};
let mut count = 0usize;
for row in row_min..=row_max {
let row_start = row * self.cols;
for col in col_min..=col_max {
let cell_idx = row_start + col;
count += (self.offsets[cell_idx + 1] - self.offsets[cell_idx]) as usize;
}
}
count
}
#[inline]
pub fn for_each_in_bounds(
&self,
@ -334,4 +354,27 @@ mod tests {
let result = grid.query(-90.0, -180.0, 90.0, 180.0);
assert_eq!(result.len(), 2);
}
#[test]
fn count_in_bounds_matches_query_len() {
let lat = vec![51.5_f32, 51.6, 51.7, 52.0];
let lon = vec![-0.1_f32, -0.1, -0.1, -0.1];
let grid = GridIndex::build(&lat, &lon, 0.01);
let bounds = (51.4, -0.2, 51.8, 0.0);
assert_eq!(
grid.count_in_bounds(bounds.0, bounds.1, bounds.2, bounds.3),
grid.query(bounds.0, bounds.1, bounds.2, bounds.3).len()
);
// Full bounds
let full = (50.0, -1.0, 53.0, 1.0);
assert_eq!(
grid.count_in_bounds(full.0, full.1, full.2, full.3),
grid.query(full.0, full.1, full.2, full.3).len()
);
// Empty bounds
assert_eq!(grid.count_in_bounds(0.0, 0.0, 1.0, 1.0), 0);
}
}