andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

More fixes

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-12 20:27:04 +00:00
parent 791bc6976b
commit 14a3555cf1
21 changed files with 549 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

11
server-rs/src/consts.rs
View file

@ -7,7 +7,7 @@ pub const H3_REQUEST_MAX: u8 = 12;
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
pub const GRID_CELL_SIZE: f32 = 0.01;
pub const MAX_POIS_PER_REQUEST: usize = 2500;
pub const MAX_POIS_PER_REQUEST: usize = 10000;
pub const MAX_CELLS_PER_REQUEST: usize = 5000;
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
pub const MAX_PROPERTIES_LIMIT: usize = 500;
@ -24,8 +24,9 @@ pub const SERVICE_CALL_TIMEOUT: u64 = 120;
/// Users without a license can only query data within these bounds.
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.42, -0.34, 51.60, 0.14);
/// Homepage demo center (lat, lng). Unlicensed hexagon requests are allowed
/// when the center of the requested bounds is within DEMO_CENTER_TOLERANCE of this point.
/// Must match DEMO_VIEW_START in ScrollStory.tsx.
/// Homepage demo center (lat, lng) and tolerance for the license bypass.
/// Hexagon requests centered within this tolerance skip the license check,
/// so the ScrollStory animation works for anonymous visitors.
/// ~0.05° ≈ 5.5 km — covers central London only.
pub const DEMO_CENTER: (f64, f64) = (51.51, -0.12);
pub const DEMO_CENTER_TOLERANCE: f64 = 1.0;
pub const DEMO_CENTER_TOLERANCE: f64 = 0.05;

11
server-rs/src/data/travel_time.rs
View file

@ -2,7 +2,7 @@ use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{bail, Context};
use anyhow::Context;
use parking_lot::Mutex;
use polars::lazy::frame::LazyFrame;
use rustc_hash::{FxHashMap, FxHashSet};
@ -167,19 +167,16 @@ impl TravelTimeStore {
}
}
// Resolve slug to actual filename (may have numeric prefix)
// Resolve slug to actual filename (may have numeric prefix).
// Reject unknown slugs rather than falling back to raw input to prevent path traversal.
let file_stem = self
.slug_to_file
.get(&key)
.map(|val| val.as_str())
.unwrap_or(slug);
.ok_or_else(|| anyhow::anyhow!("Unknown travel destination: {mode}/{slug}"))?;
let path = self
.base_dir
.join(mode)
.join(format!("{}.parquet", file_stem));
if !path.exists() {
bail!("Travel time file not found: {}", path.display());
}
let df = LazyFrame::scan_parquet(&path, Default::default())
.with_context(|| format!("Failed to scan: {}", path.display()))?

2
server-rs/src/routes/hexagons.rs
View file

@ -139,8 +139,6 @@ pub async fn get_hexagons(
let (south, west, north, east) =
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
// Allow the homepage demo: check if the center of the requested bounds
// is near the demo view center (51.51, -0.12).
let center_lat = (south + north) / 2.0;
let center_lng = (west + east) / 2.0;
let is_demo_view = (center_lat - DEMO_CENTER.0).abs() <= DEMO_CENTER_TOLERANCE

99
server-rs/src/routes/postcodes.rs
View file

@ -12,10 +12,12 @@ use tracing::info;
use crate::aggregation::Aggregator;
use crate::auth::OptionalUser;
use crate::consts::MAX_CELLS_PER_REQUEST;
use crate::data::travel_time::TravelData;
use crate::licensing::check_license_bounds;
use crate::parsing::{
bounds_intersect, parse_field_indices, parse_filters, require_bounds, row_passes_filters,
};
use crate::routes::travel_time::{parse_travel_entries, TravelTimeAgg};
use crate::state::AppState;
#[derive(Serialize)]
@ -31,6 +33,8 @@ pub struct PostcodeParams {
filters: Option<String>,
/// Comma-separated feature names to include in min/max aggregation.
fields: Option<String>,
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
travel: Option<String>,
}
/// Build a GeoJSON geometry object from postcode polygon rings.
@ -84,10 +88,41 @@ pub async fn get_postcodes(
let field_indices = parse_field_indices(params.fields.as_deref(), &state.feature_name_to_index)
.map_err(|err| (err.0, err.1).into_response())?;
// Parse travel entries
let travel_entries = params
.travel
.as_deref()
.filter(|val| !val.is_empty())
.map(parse_travel_entries)
.transpose()
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
.unwrap_or_default();
let response = tokio::task::spawn_blocking(move || -> Result<PostcodesResponse, String> {
let postcode_data = &state.postcode_data;
let t0 = std::time::Instant::now();
// Load travel time data from precomputed parquet files
let travel_data: Vec<TravelData> = if !travel_entries.is_empty() {
let store = &state.travel_time_store;
travel_entries
.iter()
.map(|entry| {
store
.get(&entry.mode, &entry.slug)
.map_err(|err| format!("Failed to load travel data: {}", err))
})
.collect::<Result<Vec<_>, _>>()?
} else {
Vec::new()
};
let has_travel = !travel_entries.is_empty();
let travel_field_keys: Vec<String> = travel_entries
.iter()
.map(|te| format!("tt_{}_{}", te.mode, te.slug))
.collect();
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let min_keys = &state.min_keys;
@ -122,9 +157,35 @@ pub async fn get_postcodes(
}
});
// Filter postcodes by travel time range (if specified)
if has_travel {
postcode_rows.retain(|&pc_idx, _rows| {
let postcode = &postcode_data.postcodes[pc_idx];
for (ti, entry) in travel_entries.iter().enumerate() {
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
let minutes = travel_data[ti].get(postcode.as_str()).map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
match minutes {
Some(mins) if (mins as f32) >= fmin && (mins as f32) <= fmax => {}
_ => return false,
}
}
}
true
});
}
// Aggregate for each postcode that has properties in bounds
// (polygon intersection check happens later when building response)
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
// Travel time aggregation per postcode
let mut travel_aggs: FxHashMap<usize, Vec<TravelTimeAgg>> = FxHashMap::default();
for (&pc_idx, rows) in &postcode_rows {
let agg = postcode_aggs
.entry(pc_idx)
@ -136,6 +197,24 @@ pub async fn get_postcodes(
agg.add_row(feature_data, row, num_features);
}
}
// Aggregate travel times for this postcode
if has_travel {
let postcode = &postcode_data.postcodes[pc_idx];
let tt_aggs = travel_aggs
.entry(pc_idx)
.or_insert_with(|| (0..travel_entries.len()).map(|_| TravelTimeAgg::new()).collect());
for (ti, entry) in travel_entries.iter().enumerate() {
if let Some(row_data) = travel_data[ti].get(postcode.as_str()) {
let minutes = if entry.use_best {
row_data.best_minutes.unwrap_or(row_data.minutes)
} else {
row_data.minutes
};
tt_aggs[ti].add(minutes as f32);
}
}
}
}
// Build response, filtering postcodes to only those whose polygon intersects query bounds
@ -218,6 +297,25 @@ pub async fn get_postcodes(
}
}
// Add travel time aggregation fields
if let Some(tt_aggs) = travel_aggs.get(&pc_idx) {
for (ti, agg) in tt_aggs.iter().enumerate() {
if agg.count > 0 {
let key = &travel_field_keys[ti];
let avg = agg.sum / agg.count as f64;
if let Some(nm) = serde_json::Number::from_f64(agg.min as f64) {
props.insert(format!("min_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(agg.max as f64) {
props.insert(format!("max_{key}"), Value::Number(nm));
}
if let Some(nm) = serde_json::Number::from_f64(avg) {
props.insert(format!("avg_{key}"), Value::Number(nm));
}
}
}
}
// Build GeoJSON Feature
let mut feature = Map::new();
feature.insert("type".into(), Value::String("Feature".into()));
@ -241,6 +339,7 @@ pub async fn get_postcodes(
bounds = format_args!("{:.6},{:.6},{:.6},{:.6}", south, west, north, east),
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
travel_entries = travel_entries.len(),
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
"GET /api/postcodes"
);

2
server-rs/src/routes/properties.rs
View file

@ -130,7 +130,7 @@ pub fn build_property(
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form",
),
duration: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Leashold/Freehold",
feature_name_to_index, feature_data, num_features, enum_values, row, "Leasehold/Freehold",
),
current_energy_rating: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating",

7
server-rs/src/routes/shorten.rs
View file

@ -95,6 +95,13 @@ pub async fn post_shorten(state: Arc<AppState>, Json(req): Json<ShortenRequest>)
}
pub async fn get_short_url(state: Arc<AppState>, Path(code): Path<String>) -> Response {
if code.is_empty()
|| code.len() > 20
|| !code.bytes().all(|b| b.is_ascii_alphanumeric())
{
return StatusCode::BAD_REQUEST.into_response();
}
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(

60
server-rs/src/routes/travel_time.rs
View file

@ -1,4 +1,62 @@
/// Per-hex-cell travel time aggregation.
/// A parsed travel time entry from the `travel` query parameter.
pub struct TravelEntry {
pub mode: String,
pub slug: String,
pub use_best: bool,
pub filter_min: Option<f32>,
pub filter_max: Option<f32>,
}
/// Parse `travel` param into a list of travel entries.
/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max`
pub fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mut entries = Vec::new();
let mut seen_keys = Vec::new();
for segment in travel_str.split('|') {
let parts: Vec<&str> = segment.split(':').collect();
if parts.len() < 2 {
return Err(format!(
"each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'",
segment
));
}
let mode = parts[0].trim().to_string();
let slug = parts[1].trim().to_string();
let use_best = parts.len() >= 3 && parts[2].trim() == "best";
let filter_offset = if use_best { 1 } else { 0 };
let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset {
let min: f32 = parts[2 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
let max: f32 = parts[3 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
(Some(min), Some(max))
} else {
(None, None)
};
let key = format!("{}:{}", mode, slug);
if seen_keys.contains(&key) {
return Err(format!("duplicate travel entry '{}'", key));
}
seen_keys.push(key);
entries.push(TravelEntry {
mode,
slug,
use_best,
filter_min,
filter_max,
});
}
Ok(entries)
}
/// Per-cell travel time aggregation.
pub struct TravelTimeAgg {
pub min: f32,
pub max: f32,