andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

lmao

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-15 22:39:49 +00:00
parent 03445188ea
commit 524580eb25
102 changed files with 36625 additions and 1295 deletions
Download patch file Download diff file Expand all files Collapse all files

334
server-rs/src/routes/hexagons.rs
View file

@ -2,19 +2,23 @@ use std::sync::Arc;
use axum::extract::Query;
use axum::http::StatusCode;
use axum::response::Json;
use axum::response::{IntoResponse, Json};
use axum::Extension;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use tracing::info;
use crate::aggregation::Aggregator;
use crate::auth::OptionalUser;
use crate::consts::MAX_CELLS_PER_REQUEST;
use crate::data::travel_time::TravelData;
use crate::licensing::check_license_bounds;
use crate::parsing::{
bounds_intersect, cell_for_row, h3_cell_bounds, needs_parent, parse_field_indices,
parse_filters, require_bounds, row_passes_filters, validate_h3_resolution,
};
use crate::routes::travel_time::fetch_travel_times;
use crate::routes::travel_time::TravelTimeAgg;
use crate::state::AppState;
#[derive(Serialize)]
@ -27,64 +31,69 @@ pub struct HexagonParams {
resolution: u8,
bounds: Option<String>,
/// Comma-separated filters: `name:min:max,...`
/// Rows must have non-NaN values within [min,max] for each filter.
filters: Option<String>,
/// Comma-separated feature names to include in min/max aggregation.
/// When present (even if empty), only listed features are aggregated and written.
/// When absent, all features are included (backward compatible).
fields: Option<String>,
/// Pipe-separated travel time entries: `lat,lon,mode|lat,lon,mode`
/// Each entry requests travel time from hex centroids to that destination via the given mode.
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
/// Each entry requests travel time aggregation for that mode+destination.
/// Optional min:max applies as a filter (exclude properties outside range).
travel: Option<String>,
}
struct TravelEntry {
lat: f64,
lon: f64,
mode: String,
slug: String,
filter_min: Option<f32>,
filter_max: Option<f32>,
}
const VALID_MODES: &[&str] = &["car", "bicycle", "walking", "transit"];
/// Parse `travel` param into a list of travel entries.
/// Format: `lat,lon,mode|lat,lon,mode`
fn parse_travel_entries(s: &str) -> Result<Vec<TravelEntry>, String> {
/// Format: `mode:slug` or `mode:slug:min:max`
fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mut entries = Vec::new();
let mut seen_modes = Vec::new();
for segment in s.split('|') {
let parts: Vec<&str> = segment.split(',').collect();
if parts.len() != 3 {
let mut seen_keys = Vec::new();
for segment in travel_str.split('|') {
let parts: Vec<&str> = segment.split(':').collect();
if parts.len() < 2 {
return Err(format!(
"each travel entry must be 'lat,lon,mode', got '{}'",
"each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'",
segment
));
}
let lat: f64 = parts[0]
.trim()
.parse()
.map_err(|_| format!("invalid travel latitude in '{}'", segment))?;
let lon: f64 = parts[1]
.trim()
.parse()
.map_err(|_| format!("invalid travel longitude in '{}'", segment))?;
let mode = parts[2].trim().to_string();
if !VALID_MODES.contains(&mode.as_str()) {
return Err(format!(
"invalid travel mode '{}', must be one of: {}",
mode,
VALID_MODES.join(", ")
));
let mode = parts[0].trim().to_string();
let slug = parts[1].trim().to_string();
let (filter_min, filter_max) = if parts.len() >= 4 {
let min: f32 = parts[2]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
let max: f32 = parts[3]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
(Some(min), Some(max))
} else {
(None, None)
};
let key = format!("{}:{}", mode, slug);
if seen_keys.contains(&key) {
return Err(format!("duplicate travel entry '{}'", key));
}
if seen_modes.contains(&mode) {
return Err(format!("duplicate travel mode '{}'", mode));
}
seen_modes.push(mode.clone());
entries.push(TravelEntry { lat, lon, mode });
seen_keys.push(key);
entries.push(TravelEntry {
mode,
slug,
filter_min,
filter_max,
});
}
Ok(entries)
}
/// Build feature maps from aggregated cell data, filtering to only cells that intersect the query bounds.
#[allow(clippy::too_many_arguments)]
fn build_feature_maps(
groups: &FxHashMap<u64, Aggregator>,
min_keys: &[String],
@ -92,7 +101,9 @@ fn build_feature_maps(
avg_keys: &[String],
num_features: usize,
indices: Option<&[usize]>,
query_bounds: (f64, f64, f64, f64), // (south, west, north, east)
query_bounds: (f64, f64, f64, f64),
travel_aggs: &[FxHashMap<u64, TravelTimeAgg>],
travel_field_keys: &[String],
) -> Vec<Map<String, Value>> {
let mut features = Vec::with_capacity(groups.len());
let (q_south, q_west, q_north, q_east) = query_bounds;
@ -143,6 +154,25 @@ fn build_feature_maps(
}
}
// Add travel time aggregation fields
for (ti, agg_map) in travel_aggs.iter().enumerate() {
if let Some(agg) = agg_map.get(&cell_id) {
if agg.count > 0 {
let key = &travel_field_keys[ti];
let avg = agg.sum / agg.count as f64;
if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) {
map.insert(format!("min_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) {
map.insert(format!("max_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(avg) {
map.insert(format!("avg_{key}"), Value::Number(nm));
}
}
}
}
features.push(map);
}
@ -151,12 +181,21 @@ fn build_feature_maps(
pub async fn get_hexagons(
state: Arc<AppState>,
Extension(user): Extension<OptionalUser>,
Query(params): Query<HexagonParams>,
) -> Result<Json<HexagonsResponse>, (StatusCode, String)> {
) -> Result<Json<HexagonsResponse>, axum::response::Response> {
let resolution = params.resolution;
validate_h3_resolution(resolution)?;
validate_h3_resolution(resolution).map_err(IntoResponse::into_response)?;
let (south, west, north, east) = require_bounds(params.bounds)?;
let (south, west, north, east) =
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
// Skip license check at low resolutions (≤5) — data is too aggregated to be
// commercially useful, and the homepage demo needs country-wide access.
if resolution > 5 {
check_license_bounds(&user.0, (south, west, north, east))
.map_err(|(_, resp)| resp)?;
}
let filters_str = params.filters.clone();
let (parsed_filters, parsed_enum_filters) = parse_filters(
@ -164,30 +203,49 @@ pub async fn get_hexagons(
&state.feature_name_to_index,
&state.data.enum_values,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err))?;
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index);
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index)
.map_err(|err| (err.0, err.1).into_response())?;
// Parse travel entries
let travel_entries = params
.travel
.as_deref()
.filter(|s| !s.is_empty())
.filter(|val| !val.is_empty())
.map(parse_travel_entries)
.transpose()
.map_err(|e| (StatusCode::BAD_REQUEST, e))?
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
.unwrap_or_default();
// Capture what we need for the R5 calls before moving state into spawn_blocking
let r5_url = state.r5_url.clone();
let http_client = state.http_client.clone();
let mut response = tokio::task::spawn_blocking(move || -> Result<HexagonsResponse, String> {
let response = tokio::task::spawn_blocking(move || -> Result<HexagonsResponse, String> {
let t0 = std::time::Instant::now();
// Load travel time data from precomputed parquet files
let travel_data: Vec<TravelData> = if !travel_entries.is_empty() {
let store = &state.travel_time_store;
travel_entries
.iter()
.map(|entry| {
store
.get(&entry.mode, &entry.slug)
.map_err(|err| format!("Failed to load travel data: {}", err))
})
.collect::<Result<Vec<_>, _>>()?
} else {
Vec::new()
};
let has_travel = !travel_entries.is_empty();
let travel_field_keys: Vec<String> = travel_entries
.iter()
.map(|te| format!("tt_{}_{}", te.mode, te.slug))
.collect();
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let (pc_interner, pc_keys) = state.data.postcode_parts();
let min_keys = &state.min_keys;
let max_keys = &state.max_keys;
let avg_keys = &state.avg_keys;
@ -198,49 +256,70 @@ pub async fn get_hexagons(
let need_parent = needs_parent(resolution);
let mut groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
let mut travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> =
(0..travel_entries.len()).map(|_| FxHashMap::default()).collect();
// Hoist has_selective branch outside the hot loop to avoid per-row branching
if let Some(sel_indices) = field_indices.as_deref() {
state
.grid
.for_each_in_bounds(south, west, north, east, |row_idx| {
let row = row_idx as usize;
if !row_passes_filters(
row,
&parsed_filters,
&parsed_enum_filters,
feature_data,
num_features,
) {
return;
// Main aggregation loop
let aggregate_row =
|row: usize,
groups: &mut FxHashMap<u64, Aggregator>,
travel_aggs: &mut [FxHashMap<u64, TravelTimeAgg>]| {
// Regular filters
if !row_passes_filters(
row,
&parsed_filters,
&parsed_enum_filters,
feature_data,
num_features,
) {
return;
}
// Travel time filter: check each entry with a range
let mut travel_minutes: Vec<Option<i16>> = Vec::new();
if has_travel {
let postcode = pc_interner.resolve(&pc_keys[row]);
travel_minutes.reserve(travel_entries.len());
for (ti, entry) in travel_entries.iter().enumerate() {
let minutes = travel_data[ti].get(postcode).copied();
travel_minutes.push(minutes);
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
match minutes {
Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => return, // Filtered out
}
}
}
let cell_id = cell_for_row(row, precomputed, h3_res, need_parent);
let aggregation = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
}
let cell_id = cell_for_row(row, precomputed, h3_res, need_parent);
// Aggregate regular features
let aggregation = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
if let Some(sel_indices) = field_indices.as_deref() {
aggregation.add_row_selective(feature_data, row, num_features, sel_indices);
});
} else {
state
.grid
.for_each_in_bounds(south, west, north, east, |row_idx| {
let row = row_idx as usize;
if !row_passes_filters(
row,
&parsed_filters,
&parsed_enum_filters,
feature_data,
num_features,
) {
return;
}
let cell_id = cell_for_row(row, precomputed, h3_res, need_parent);
let aggregation = groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
} else {
aggregation.add_row(feature_data, row, num_features);
});
}
}
// Aggregate travel time
for (ti, minutes) in travel_minutes.iter().enumerate() {
if let Some(mins) = minutes {
let agg = travel_aggs[ti]
.entry(cell_id)
.or_insert_with(TravelTimeAgg::new);
agg.add(*mins as f32);
}
}
};
state
.grid
.for_each_in_bounds(south, west, north, east, |row_idx| {
aggregate_row(row_idx as usize, &mut groups, &mut travel_aggs);
});
let t_agg = t0.elapsed();
@ -252,6 +331,8 @@ pub async fn get_hexagons(
num_features,
field_indices.as_deref(),
(south, west, north, east),
&travel_aggs,
&travel_field_keys,
);
let truncated = features.len() > MAX_CELLS_PER_REQUEST;
@ -268,6 +349,7 @@ pub async fn get_hexagons(
bounds = format_args!("{:.4},{:.4},{:.4},{:.4}", south, west, north, east),
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
travel_entries = travel_entries.len(),
agg_ms = format_args!("{:.1}", t_agg.as_secs_f64() * 1000.0),
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
"GET /api/hexagons"
@ -276,76 +358,8 @@ pub async fn get_hexagons(
Ok(HexagonsResponse { features })
})
.await
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?;
// If travel entries were requested and R5 is configured, fetch travel times concurrently.
if !travel_entries.is_empty() {
let url = r5_url.as_deref().ok_or((
StatusCode::SERVICE_UNAVAILABLE,
"Travel time queries require routing service (R5_URL not configured)".into(),
))?;
// Collect hex centroids
let origins: Vec<[f64; 2]> = response
.features
.iter()
.map(|f| {
let lat = f
.get("lat")
.and_then(|v| v.as_f64())
.expect("lat must be present in feature map");
let lon = f
.get("lon")
.and_then(|v| v.as_f64())
.expect("lon must be present in feature map");
[lat, lon]
})
.collect();
// Fire concurrent R5 calls for each travel entry
let mut handles = Vec::with_capacity(travel_entries.len());
for entry in &travel_entries {
let client = http_client.clone();
let url = url.to_string();
let origins = origins.clone();
let dest = [entry.lat, entry.lon];
let mode = entry.mode.clone();
handles.push(tokio::spawn(async move {
fetch_travel_times(&client, &url, origins, dest, &mode).await
}));
}
let mut results = Vec::with_capacity(handles.len());
for handle in handles {
results.push(handle.await);
}
for (entry, result) in travel_entries.iter().zip(results) {
let travel_times = result
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
.map_err(|err| (StatusCode::BAD_GATEWAY, err))?;
let field_name = format!("travel_time_{}", entry.mode);
for (feature, tt) in response.features.iter_mut().zip(&travel_times) {
match tt {
Some(minutes) => {
if let Some(num) = serde_json::Number::from_f64(*minutes) {
feature.insert(field_name.clone(), Value::Number(num));
}
}
None => {
feature.insert(field_name.clone(), Value::Null);
}
}
}
info!(
hexagons = response.features.len(),
destination = format_args!("{},{}", entry.lat, entry.lon),
mode = entry.mode,
"Travel times merged"
);
}
}
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response())?
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error).into_response())?;
Ok(Json(response))
}