new demo mode & tenure
Some checks failed
Build and publish Docker image / build-and-push (push) Successful in 8m43s
CI / Check (push) Failing after 8m49s

This commit is contained in:
Andras Schmelczer 2026-06-17 07:54:30 +01:00
parent 7656f24544
commit 4a0f00f2a4
64 changed files with 2875 additions and 338 deletions

View file

@ -3,9 +3,9 @@ pub const QUANT_SCALE: f32 = 65534.0;
pub const HISTOGRAM_BINS: usize = 100;
pub const H3_PRECOMPUTE_MAX: u8 = 12;
pub const H3_PRECOMPUTE_MAX: u8 = 9;
pub const H3_REQUEST_MIN: u8 = 4;
pub const H3_REQUEST_MAX: u8 = 12;
pub const H3_REQUEST_MAX: u8 = 9;
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
@ -15,6 +15,12 @@ pub const MAX_POIS_PER_REQUEST: usize = 3000;
pub const PROPERTIES_LIMIT: usize = 100;
pub const PLACES_LIMIT: usize = 20;
/// Enum feature whose count is an attribute of the postcode/area as a whole
/// rather than of the filter-matching subset: it is always computed over every
/// property in the area so applying filters never changes it. Must match the
/// feature name in `features.rs`.
pub const FORMER_COUNCIL_HOUSE_FEATURE: &str = "Former council house";
pub const PRICE_HISTORY_POINTS_LIMIT: usize = 5000;
pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02;
@ -26,9 +32,29 @@ pub const AI_FILTERS_WEEKLY_TOKEN_LIMIT: u64 = 10_000_000;
/// Timeout for outbound HTTP service calls (seconds).
pub const SERVICE_CALL_TIMEOUT: u64 = 120;
/// Demo free zone bounds (south, west, north, east) — inner London, roughly zone 1.
/// Users without a license can only query data within these bounds.
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.44, -0.31, 51.59, 0.05);
/// Half-spans of the demo free zone box, in degrees. The box is built this wide
/// around the visitor's approximate (IP-derived) location — or the fallback
/// centre below when the IP can't be geolocated. Sized to match the original
/// inner-London "zone 1" box (≈0.15° lat × 0.36° lon).
pub const FREE_ZONE_HALF_LAT: f64 = 0.075;
pub const FREE_ZONE_HALF_LON: f64 = 0.18;
/// Fallback demo free-zone centre (Canary Wharf), used when the visitor hasn't
/// supplied a demo centre — i.e. they declined the "check your area" prompt or no
/// GPS fix was available. When they consent, the browser sends their own centre.
pub const FREE_ZONE_FALLBACK_LAT: f64 = 51.5054;
pub const FREE_ZONE_FALLBACK_LON: f64 = -0.0235;
/// Demo (unlicensed) users may apply at most this many filters. Enforced both
/// client-side (UX) and server-side (anti-DoS); must match the frontend constant.
pub const DEMO_MAX_FILTERS: usize = 5;
/// Sliding-window rate limit for unlicensed traffic to `/api/` endpoints, keyed by
/// account id (preferred) or client IP. Generous enough for active demo use, tight
/// enough to blunt tile-by-tile scraping. Licensed users/admins are exempt.
pub const DEMO_RATE_LIMIT_MAX: usize = 600;
pub const DEMO_RATE_LIMIT_WINDOW_SECS: u64 = 60;
/// Soft cap on tracked rate-limit keys before stale ones are pruned.
pub const DEMO_RATE_LIMIT_MAX_KEYS: usize = 50_000;
pub const SHARE_CACHE_TTL_SECS: u64 = 300;
pub const SHARE_CACHE_MAX_ENTRIES: usize = 1024;

View file

@ -67,6 +67,6 @@ pub use poi::{resolve_poi_category_filter, POICategoryGroup, POIData, SchoolMeta
pub use postcodes::{OutcodeData, PostcodeData};
pub use property::{
precompute_h3, FeatureStats, Histogram, HistoricalPrice, PostcodePoiMetrics, PropertyData,
QuantRef, RenovationEvent,
QuantRef, RenovationEvent, TenureEvent,
};
pub use travel_time::{slugify, TravelTimeStore};

View file

@ -251,6 +251,26 @@ pub(super) fn address_search_tokens(text: &str) -> Vec<String> {
tokens
}
/// Search tokens for a property row, covering its display address plus an
/// alternative spelling (`alt`, the EPC form) when distinct — so the property is
/// findable by either form. The result is deduped: each token appears at most
/// once, which the inverted-index posting lists rely on (a token must post a
/// given row exactly once to keep those lists strictly ascending and unique).
pub(super) fn merged_address_search_tokens(display: &str, alt: Option<&str>) -> Vec<String> {
let mut tokens = address_search_tokens(display);
if let Some(alt) = alt {
let alt = alt.trim();
if !alt.is_empty() && alt != display.trim() {
for token in address_search_tokens(alt) {
if !tokens.contains(&token) {
tokens.push(token);
}
}
}
}
tokens
}
fn is_address_search_token(token: &str) -> bool {
if looks_like_postcode_fragment(token) {
return false;
@ -929,6 +949,60 @@ mod tests {
assert_eq!(parsed.text_groups[1].alternatives, vec!["cour".to_string()]);
}
#[test]
fn merged_tokens_index_both_address_forms() {
// A property whose price-paid form is "10 Park Road" but whose EPC form
// names the building ("Kingswood House 10 Park Road") must be findable by
// the building name too: the distinctive "kingswood" token comes only
// from the EPC form.
let display_only = address_search_tokens("10 Park Road");
assert!(!display_only.contains(&"kingswood".to_string()));
let merged =
merged_address_search_tokens("10 Park Road", Some("Kingswood House 10 Park Road"));
assert!(merged.contains(&"kingswood".to_string()));
// Tokens shared by both forms are still present and not duplicated.
assert!(merged.contains(&"park".to_string()));
assert!(merged.contains(&"road".to_string()));
assert!(merged.contains(&"10".to_string()));
// No token repeats — posting-list uniqueness depends on this.
let mut deduped = merged.clone();
deduped.sort_unstable();
deduped.dedup();
assert_eq!(deduped.len(), merged.len());
}
#[test]
fn merged_tokens_ignore_redundant_or_missing_alt_form() {
let baseline = address_search_tokens("12 Baker Street");
// An EPC form identical to the display adds nothing.
assert_eq!(
merged_address_search_tokens("12 Baker Street", Some("12 Baker Street")),
baseline
);
// A blank EPC form (whitespace) and an absent one are both no-ops.
assert_eq!(
merged_address_search_tokens("12 Baker Street", Some(" ")),
baseline
);
assert_eq!(
merged_address_search_tokens("12 Baker Street", None),
baseline
);
}
#[test]
fn merged_tokens_handle_epc_only_display() {
// EPC-only dwellings have no price-paid form, so the EPC address IS the
// display; passing it as both arguments must not duplicate its tokens.
let epc = "Flat 4 Rosewood Court";
assert_eq!(
merged_address_search_tokens(epc, Some(epc)),
address_search_tokens(epc)
);
}
#[test]
fn address_search_tokens_keep_actual_address_terms_for_scoring() {
let tokens = address_search_tokens("Flat 2, 10 Downing Cour");

View file

@ -14,11 +14,11 @@ use crate::consts::{NAN_U16, QUANT_SCALE};
use crate::features::{self, Bounds};
use super::address_search::{
address_search_tokens, build_address_prefix_index, is_address_candidate_token,
build_address_prefix_index, is_address_candidate_token, merged_address_search_tokens,
};
use super::poi_metrics::{PostcodePoiMetrics, NO_POI_METRIC_ROW};
use super::stats::{column_to_f32_vec, compute_feature_stats, FeatureStats, Histogram};
use super::{HistoricalPrice, PropertyData, RenovationEvent};
use super::{HistoricalPrice, PropertyData, RenovationEvent, TenureEvent};
const MISSING_COORDINATE_SAMPLE_LIMIT: usize = 10;
const COUNTRY_COLUMN_CANDIDATES: &[&str] = &[
@ -330,6 +330,10 @@ impl PropertyData {
if has_historical_prices {
select_exprs.push(col("historical_prices"));
}
let has_tenure_history = schema.get("tenure_history").is_some();
if has_tenure_history {
select_exprs.push(col("tenure_history"));
}
let df = combined_lf
.filter(col("lat").is_not_null().and(col("lon").is_not_null()))
.select(select_exprs)
@ -439,7 +443,6 @@ impl PropertyData {
.collect()
};
let address_raw = extract_string_col(&df, "Address per Property Register")?;
let postcode_raw = extract_string_col(&df, "Postcode")?;
// Extract optional string columns
@ -468,6 +471,13 @@ impl PropertyData {
};
let property_sub_type_raw = extract_optional_string_col(&df, "Property sub-type")?;
let price_qualifier_raw = extract_optional_string_col(&df, "Price qualifier")?;
// Display + search addresses. The price-paid form ("Address per Property
// Register") is the primary display address; the EPC form ("Address per
// EPC") is an alternative spelling indexed alongside it so a property is
// findable by either. For EPC-only dwellings (no Land Registry sale) the
// price-paid form is null, so the EPC form becomes the display address.
let pp_address_raw = extract_optional_string_col(&df, "Address per Property Register")?;
let epc_address_raw = extract_optional_string_col(&df, "Address per EPC")?;
tracing::info!("Building enum features");
// enum_col_major: Vec<(values_list, encoded_as_f32)>
@ -675,6 +685,57 @@ impl PropertyData {
FxHashMap::default()
};
// Extract tenure_history: List<Struct{year: i32, status: str}>
let mut tenure_raw: FxHashMap<u32, Vec<TenureEvent>> = if has_tenure_history {
tracing::info!("Extracting tenure history");
let tenure_col = df
.column("tenure_history")
.context("Missing tenure_history column")?;
let list_ca = tenure_col
.list()
.context("tenure_history is not a list column")?;
let mut history: FxHashMap<u32, Vec<TenureEvent>> = FxHashMap::default();
for old_row in 0..row_count {
if let Some(inner) = list_ca.get_as_series(old_row) {
if inner.is_empty() {
continue;
}
let structs = inner
.struct_()
.context("tenure_history inner is not a struct")?;
let years = structs
.field_by_name("year")
.context("Missing 'year' field in tenure_history struct")?;
let statuses = structs
.field_by_name("status")
.context("Missing 'status' field in tenure_history struct")?;
let mut row_events = Vec::new();
for idx in 0..inner.len() {
let year = years.get(idx).context("Failed to get year value")?;
let status = statuses.get(idx).context("Failed to get status value")?;
if let (AnyValue::Int32(yr), AnyValue::String(st)) = (&year, &status) {
row_events.push(TenureEvent {
year: *yr,
status: st.to_string(),
});
}
}
if !row_events.is_empty() {
history.insert(old_row as u32, row_events);
}
}
}
tracing::info!(
properties_with_tenure_changes = history.len(),
"Tenure history extracted"
);
history
} else {
FxHashMap::default()
};
// Free the projected joined frame before building the row-major matrix.
drop(df);
@ -712,9 +773,21 @@ impl PropertyData {
})
.context("Required numeric column 'Last known price' not configured")?;
// Build contiguous address buffer and address search index (permuted)
// Build contiguous address buffer and address search index (permuted).
// Each row's posting lists cover BOTH address spellings we hold — the
// price-paid form and the EPC form — so a property is findable by either;
// the display address prefers the price-paid form, falling back to the EPC
// form for never-sold (EPC-only) dwellings whose price-paid form is null.
tracing::info!("Building interned strings");
let total_addr_bytes: usize = address_raw.iter().map(|text| text.len()).sum();
// Display address for a row, preferring the price-paid form and falling
// back to the EPC form (so EPC-only dwellings still have a display string).
let coalesced_display = |old_row: usize| -> &str {
match pp_address_raw[old_row].as_deref() {
Some(pp) if !pp.is_empty() => pp,
_ => epc_address_raw[old_row].as_deref().unwrap_or(""),
}
};
let total_addr_bytes: usize = (0..row_count).map(|row| coalesced_display(row).len()).sum();
let mut address_buffer = String::with_capacity(total_addr_bytes);
let mut address_offsets = Vec::with_capacity(row_count);
let mut address_lengths = Vec::with_capacity(row_count);
@ -724,14 +797,19 @@ impl PropertyData {
let mut address_search_token_offsets = Vec::with_capacity(row_count);
let mut address_search_token_lengths = Vec::with_capacity(row_count);
for (new_row, &perm_index) in perm.iter().enumerate() {
let addr = &address_raw[perm_index as usize];
let old_row = perm_index as usize;
let display = coalesced_display(old_row);
let offset = address_buffer.len() as u32;
let length = addr.len().min(u16::MAX as usize) as u16;
let length = display.len().min(u16::MAX as usize) as u16;
address_offsets.push(offset);
address_lengths.push(length);
address_buffer.push_str(&addr[..length as usize]);
address_buffer.push_str(&display[..length as usize]);
let search_tokens = address_search_tokens(addr);
// Tokens cover the display address plus the EPC-form spelling when
// distinct, so the property is findable by either form (deduped so
// each token posts this row exactly once).
let search_tokens =
merged_address_search_tokens(display, epc_address_raw[old_row].as_deref());
let token_offset = address_search_token_keys.len() as u32;
let token_length = search_tokens.len().min(u16::MAX as usize) as u16;
address_search_token_offsets.push(token_offset);
@ -834,6 +912,18 @@ impl PropertyData {
map
};
// Re-key tenure_history by permuted row index
let tenure_history: FxHashMap<u32, Vec<TenureEvent>> = {
let mut map =
FxHashMap::with_capacity_and_hasher(tenure_raw.len(), Default::default());
for (new_row, &old_row) in perm.iter().enumerate() {
if let Some(events) = tenure_raw.remove(&old_row) {
map.insert(new_row as u32, events);
}
}
map
};
// Permute optional string columns into sparse HashMaps
let property_sub_type: FxHashMap<u32, String> = {
let mut map = FxHashMap::default();
@ -968,6 +1058,7 @@ impl PropertyData {
approx_build_date_bits,
renovation_history,
historical_prices,
tenure_history,
property_sub_type,
price_qualifier,
})

View file

@ -40,6 +40,16 @@ pub struct HistoricalPrice {
pub price: i64,
}
/// A point on the property's tenure timeline: the year a certificate first
/// recorded a new occupancy `status` ("Owner-occupied" / "Rented (private)" /
/// "Rented (social)"). Derived per-EPC and reduced to change points by the
/// pipeline, so this surfaces *when* a home was let out vs lived in.
#[derive(Serialize, Clone)]
pub struct TenureEvent {
pub year: i32,
pub status: String,
}
pub struct PropertyData {
pub lat: Vec<f32>,
pub lon: Vec<f32>,
@ -100,6 +110,9 @@ pub struct PropertyData {
/// Per-row historical sale transactions (Land Registry price-paid).
/// Keyed by (permuted) row index. Only rows with prices are present.
historical_prices: FxHashMap<u32, Vec<HistoricalPrice>>,
/// Per-row tenure timeline (owner-occupied <-> rented change points from
/// EPC). Keyed by (permuted) row index. Only rows with changes are present.
tenure_history: FxHashMap<u32, Vec<TenureEvent>>,
property_sub_type: FxHashMap<u32, String>,
price_qualifier: FxHashMap<u32, String>,
}
@ -153,6 +166,14 @@ impl PropertyData {
.unwrap_or(&[])
}
/// Get tenure timeline events for a given row (empty slice if none).
pub fn tenure_history(&self, row: usize) -> &[TenureEvent] {
self.tenure_history
.get(&(row as u32))
.map(|v| v.as_slice())
.unwrap_or(&[])
}
/// Get property sub-type for a given row.
pub fn property_sub_type(&self, row: usize) -> Option<&str> {
self.property_sub_type
@ -231,6 +252,7 @@ impl PropertyData {
approx_build_date_bits: Vec::new(),
renovation_history: FxHashMap::default(),
historical_prices: FxHashMap::default(),
tenure_history: FxHashMap::default(),
property_sub_type: FxHashMap::default(),
price_qualifier: FxHashMap::default(),
}

127
server-rs/src/demo_zone.rs Normal file
View file

@ -0,0 +1,127 @@
//! Per-visitor demo "free zone": the small area an unlicensed visitor may explore.
//!
//! The visitor's browser chooses the centre — their GPS location (with consent) or
//! a Canary Wharf default — and sends it on each request as `demoLat`/`demoLon`
//! query params. We build a fixed-size box around that centre for the licensing
//! check. Absent/invalid params fall back to Canary Wharf.
//!
//! The centre is client-supplied and therefore spoofable: this is a soft demo gate
//! backed by the rate-limit / filter-cap guards in `ratelimit.rs`, not a hard
//! security boundary. (We deliberately removed IP-based geolocation in favour of
//! this explicit, consent-based flow.)
use axum::extract::Request;
use axum::middleware::Next;
use axum::response::Response;
use serde::Serialize;
use url::form_urlencoded;
use crate::consts::{
FREE_ZONE_FALLBACK_LAT, FREE_ZONE_FALLBACK_LON, FREE_ZONE_HALF_LAT, FREE_ZONE_HALF_LON,
};
/// A demo free-zone bounding box (degrees).
#[derive(Clone, Copy, Debug, Serialize)]
pub struct FreeZone {
pub south: f64,
pub west: f64,
pub north: f64,
pub east: f64,
}
/// A visitor's resolved demo zone, inserted into request extensions by
/// [`demo_zone_middleware`] for every request.
#[derive(Clone, Copy, Debug)]
pub struct DemoZone {
pub free_zone: FreeZone,
}
/// Build the demo free-zone box centred on `(lat, lon)`, keeping a fixed size and
/// clamping to valid lat/lon ranges.
pub fn free_zone_around(lat: f64, lon: f64) -> FreeZone {
FreeZone {
south: (lat - FREE_ZONE_HALF_LAT).max(-90.0),
north: (lat + FREE_ZONE_HALF_LAT).min(90.0),
west: (lon - FREE_ZONE_HALF_LON).max(-180.0),
east: (lon + FREE_ZONE_HALF_LON).min(180.0),
}
}
/// Read the visitor's chosen demo centre from the `demoLat`/`demoLon` query params.
fn demo_center_from_query(query: Option<&str>) -> Option<(f64, f64)> {
let query = query?;
let mut lat: Option<f64> = None;
let mut lon: Option<f64> = None;
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
match key.as_ref() {
"demoLat" => lat = value.parse().ok(),
"demoLon" => lon = value.parse().ok(),
_ => {}
}
}
let (lat, lon) = (lat?, lon?);
if lat.is_finite()
&& lon.is_finite()
&& (-90.0..=90.0).contains(&lat)
&& (-180.0..=180.0).contains(&lon)
{
Some((lat, lon))
} else {
None
}
}
/// Resolve a request's demo zone from its query params, falling back to Canary Wharf.
pub fn resolve_demo_zone(query: Option<&str>) -> DemoZone {
let (lat, lon) = demo_center_from_query(query)
.unwrap_or((FREE_ZONE_FALLBACK_LAT, FREE_ZONE_FALLBACK_LON));
DemoZone {
free_zone: free_zone_around(lat, lon),
}
}
/// Middleware that resolves the visitor's [`DemoZone`] and stashes it in request
/// extensions, mirroring how `auth_middleware` provides `OptionalUser`.
pub async fn demo_zone_middleware(req: Request, next: Next) -> Response {
let zone = resolve_demo_zone(req.uri().query());
let (mut parts, body) = req.into_parts();
parts.extensions.insert(zone);
next.run(Request::from_parts(parts, body)).await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fallback_box_is_canary_wharf_sized_and_centred() {
let fz = resolve_demo_zone(None).free_zone;
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
assert!((((fz.west + fz.east) / 2.0) - FREE_ZONE_FALLBACK_LON).abs() < 1e-9);
}
#[test]
fn reads_client_supplied_centre() {
let fz = resolve_demo_zone(Some("demoLat=53.4808&demoLon=-2.2426&foo=bar")).free_zone;
assert!((((fz.south + fz.north) / 2.0) - 53.4808).abs() < 1e-9);
assert!((((fz.west + fz.east) / 2.0) - (-2.2426)).abs() < 1e-9);
}
#[test]
fn rejects_out_of_range_or_missing_coords() {
// Out of range → fallback.
let fz = resolve_demo_zone(Some("demoLat=999&demoLon=0")).free_zone;
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
// Only one coord → fallback.
let fz = resolve_demo_zone(Some("demoLat=53.0")).free_zone;
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
}
#[test]
fn box_keeps_a_constant_size_when_relocated() {
let a = free_zone_around(51.5, -0.1);
let b = free_zone_around(55.0, -3.0);
assert!(((a.north - a.south) - (b.north - b.south)).abs() < 1e-9);
assert!(((a.east - a.west) - (b.east - b.west)).abs() < 1e-9);
}
}

View file

@ -10,9 +10,10 @@ use url::form_urlencoded;
use crate::auth::PocketBaseUser;
use crate::consts::{
FREE_ZONE_BOUNDS, MAX_SHARE_LAT_SPAN, MAX_SHARE_LON_SPAN, MAX_SHARE_ZOOM, MIN_SHARE_ZOOM,
SHARE_CACHE_MAX_ENTRIES, SHARE_CACHE_TTL_SECS,
MAX_SHARE_LAT_SPAN, MAX_SHARE_LON_SPAN, MAX_SHARE_ZOOM, MIN_SHARE_ZOOM, SHARE_CACHE_MAX_ENTRIES,
SHARE_CACHE_TTL_SECS,
};
use crate::demo_zone::FreeZone;
use crate::pocketbase::get_superuser_token;
use crate::state::AppState;
@ -225,12 +226,14 @@ pub fn is_valid_share_bounds(bounds: ShareBounds) -> bool {
/// Check whether the user is allowed to query data at the given bounds.
/// Licensed users and admins bypass the check entirely.
/// Free/anonymous users get 403 unless the bounds fall inside the free zone
/// or inside the bbox granted by a valid share code.
/// Free/anonymous users get 403 unless the bounds fall inside their free zone
/// (a box centred on their IP-approximate location) or inside the bbox granted
/// by a valid share code.
#[allow(clippy::result_large_err)]
pub fn check_license_bounds(
user: &Option<PocketBaseUser>,
bounds: (f64, f64, f64, f64),
free_zone: FreeZone,
share_bounds: Option<ShareBounds>,
) -> Result<(), axum::response::Response> {
if let Some(u) = user {
@ -240,9 +243,12 @@ pub fn check_license_bounds(
}
let (south, west, north, east) = bounds;
let (fz_south, fz_west, fz_north, fz_east) = FREE_ZONE_BOUNDS;
if south >= fz_south && west >= fz_west && north <= fz_north && east <= fz_east {
if south >= free_zone.south
&& west >= free_zone.west
&& north <= free_zone.north
&& east <= free_zone.east
{
return Ok(());
}
@ -256,10 +262,10 @@ pub fn check_license_bounds(
"error": "license_required",
"message": "A license is required to view data outside the demo area",
"free_zone": {
"south": fz_south,
"west": fz_west,
"north": fz_north,
"east": fz_east,
"south": free_zone.south,
"west": free_zone.west,
"north": free_zone.north,
"east": free_zone.east,
}
});
@ -272,9 +278,10 @@ pub fn check_license_point(
user: &Option<PocketBaseUser>,
lat: f64,
lon: f64,
free_zone: FreeZone,
share_bounds: Option<ShareBounds>,
) -> Result<(), axum::response::Response> {
check_license_bounds(user, (lat, lon, lat, lon), share_bounds)
check_license_bounds(user, (lat, lon, lat, lon), free_zone, share_bounds)
}
#[cfg(test)]

View file

@ -7,6 +7,7 @@ mod bugsink;
mod checkout_sessions;
mod consts;
mod data;
mod demo_zone;
mod features;
mod language;
mod licensing;
@ -15,6 +16,7 @@ mod og_middleware;
pub mod parsing;
mod pocketbase;
mod pocketbase_locks;
mod ratelimit;
mod routes;
mod state;
pub mod utils;
@ -720,6 +722,7 @@ async fn main() -> anyhow::Result<()> {
stripe_webhook_secret: cli.stripe_webhook_secret,
stripe_referral_coupon_id: cli.stripe_referral_coupon_id,
bugsink_frontend_config,
demo_rate_limiter: Arc::new(ratelimit::DemoRateLimiter::new()),
};
let shared = Arc::new(SharedState::new(app_state));
@ -999,6 +1002,8 @@ async fn main() -> anyhow::Result<()> {
api
}
.layer(middleware::from_fn(metrics::track_metrics))
.layer(middleware::from_fn(ratelimit::demo_guard_middleware))
.layer(middleware::from_fn(demo_zone::demo_zone_middleware))
.layer(middleware::from_fn(auth::auth_middleware))
.layer(middleware::from_fn(
move |req: axum::extract::Request, next: middleware::Next| {

256
server-rs/src/ratelimit.rs Normal file
View file

@ -0,0 +1,256 @@
//! Defense-in-depth guards for unlicensed ("demo") API traffic: a per-key rate
//! limit and a server-side filter-count cap. These complement the client-side UX
//! limits so a scripted client can't trivially bypass them. Licensed users and
//! admins are exempt.
//!
//! This is a soft anti-abuse measure, not a hard security boundary: the rate-limit
//! key falls back to the (spoofable) client IP for anonymous users, and the demo
//! gate is best-effort by design (see demo_zone.rs). Logged-in free accounts are keyed
//! by their stable account id, which spoofing the IP header does not evade.
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::http::HeaderMap;
use axum::extract::{ConnectInfo, Request};
use axum::http::StatusCode;
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use parking_lot::Mutex;
use rustc_hash::FxHashMap;
use serde_json::json;
use url::form_urlencoded;
use crate::auth::OptionalUser;
use crate::consts::{
DEMO_MAX_FILTERS, DEMO_RATE_LIMIT_MAX, DEMO_RATE_LIMIT_MAX_KEYS, DEMO_RATE_LIMIT_WINDOW_SECS,
};
use crate::state::AppState;
/// Extract the client IP, preferring reverse-proxy/CDN headers (we sit behind one
/// in production) and falling back to the socket peer. Client-supplied and
/// spoofable — only used as a coarse rate-limit key for anonymous visitors.
fn client_ip(headers: &HeaderMap, peer: Option<IpAddr>) -> Option<IpAddr> {
let from_header = |name: &str, take_first: bool| -> Option<IpAddr> {
let raw = headers.get(name)?.to_str().ok()?;
let candidate = if take_first {
raw.split(',').next().unwrap_or(raw)
} else {
raw
};
candidate.trim().parse().ok()
};
from_header("cf-connecting-ip", false)
.or_else(|| from_header("x-forwarded-for", true))
.or_else(|| from_header("x-real-ip", false))
.or(peer)
}
/// Sliding-window request counter keyed by account id or client IP.
pub struct DemoRateLimiter {
windows: Mutex<FxHashMap<String, Vec<Instant>>>,
}
impl DemoRateLimiter {
pub fn new() -> Self {
Self {
windows: Mutex::new(FxHashMap::default()),
}
}
/// Record a hit for `key`; returns `false` when the per-window limit is
/// exceeded (caller should reject with 429).
pub fn check(&self, key: &str) -> bool {
let now = Instant::now();
let window = Duration::from_secs(DEMO_RATE_LIMIT_WINDOW_SECS);
let mut map = self.windows.lock();
// Bound memory: when the table grows large, drop keys with no recent hits.
// If even that can't get us back under the cap (e.g. a spoofed-IP flood
// minting a fresh key per request), clear the table outright. That resets
// everyone's window — acceptable under attack — and keeps both the size and
// the cost of this scan bounded (it then won't re-run for ~MAX_KEYS inserts,
// instead of scanning O(n) on every request once full).
if map.len() > DEMO_RATE_LIMIT_MAX_KEYS {
map.retain(|_, hits| hits.iter().any(|t| now.duration_since(*t) < window));
if map.len() > DEMO_RATE_LIMIT_MAX_KEYS {
map.clear();
}
}
let hits = map.entry(key.to_string()).or_default();
hits.retain(|t| now.duration_since(*t) < window);
if hits.len() >= DEMO_RATE_LIMIT_MAX {
return false;
}
hits.push(now);
true
}
}
impl Default for DemoRateLimiter {
fn default() -> Self {
Self::new()
}
}
/// Whether a request is internal/direct rather than from a public visitor via the
/// edge proxy. Internal callers (the screenshot/OG service, health checks, local
/// dev) reach the server container directly: a private/loopback socket peer and no
/// edge-proxy forwarding headers. We exempt them so demo guards never throttle our
/// own services. (Behind the public proxy, real visitors carry forwarding headers.)
fn is_internal_request(headers: &HeaderMap, peer: Option<IpAddr>) -> bool {
let has_forwarding = headers.contains_key("cf-connecting-ip")
|| headers.contains_key("x-forwarded-for")
|| headers.contains_key("x-real-ip");
if has_forwarding {
return false;
}
match peer {
Some(IpAddr::V4(v4)) => {
v4.is_private() || v4.is_loopback() || v4.is_link_local() || v4.is_unspecified()
}
Some(IpAddr::V6(v6)) => {
v6.is_loopback()
|| v6.is_unspecified()
|| (v6.segments()[0] & 0xfe00) == 0xfc00 // fc00::/7 unique-local
|| (v6.segments()[0] & 0xffc0) == 0xfe80 // fe80::/10 link-local
}
None => true,
}
}
/// Count non-empty `;;`-separated entries in the `filters` query parameter.
fn filter_count(query: &str) -> usize {
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
if key == "filters" {
return value.split(";;").filter(|entry| !entry.trim().is_empty()).count();
}
}
0
}
/// Whether the request carries a non-empty `share` code. Such requests view a shared
/// dashboard whose saved filter set may legitimately exceed the demo cap, so they're
/// exempt from the filter cap — the actual share-bounds authorization still runs in
/// the handler.
fn has_share_code(query: &str) -> bool {
form_urlencoded::parse(query.as_bytes())
.any(|(key, value)| key == "share" && !value.trim().is_empty())
}
/// Middleware applying the demo filter cap and rate limit to unlicensed `/api/`
/// traffic. Must run after `auth_middleware` (for `OptionalUser`) and the state
/// injection layer (for `AppState`).
pub async fn demo_guard_middleware(req: Request, next: Next) -> Response {
let path = req.uri().path();
// Map/overlay tiles aren't property-data pulls, and the basemap visibly breaks if
// they're throttled, so the demo guards only cover the data endpoints.
if !path.starts_with("/api/")
|| path.starts_with("/api/tiles/")
|| path.starts_with("/api/overlays/")
{
return next.run(req).await;
}
// Licensed users and admins bypass demo guards entirely.
let user = req.extensions().get::<OptionalUser>().and_then(|u| u.0.clone());
if let Some(u) = &user {
if u.is_admin || u.subscription == "licensed" {
return next.run(req).await;
}
}
let peer = req
.extensions()
.get::<ConnectInfo<SocketAddr>>()
.map(|info| info.0.ip());
// Exempt our own internal services (screenshots/OG, health checks, dev).
if is_internal_request(req.headers(), peer) {
return next.run(req).await;
}
// Server-side filter cap (mirrors the client demo limit; blunts oversized
// aggregation requests). Share recipients are exempt — their saved view may
// legitimately carry >5 filters and the share grant is checked in the handler.
if let Some(query) = req.uri().query() {
if !has_share_code(query) && filter_count(query) > DEMO_MAX_FILTERS {
return (
StatusCode::BAD_REQUEST,
axum::Json(json!({
"error": "filter_limit",
"message": format!("The demo is limited to {DEMO_MAX_FILTERS} filters"),
})),
)
.into_response();
}
}
// Rate limit per account (stable, not IP-spoofable) or, for anonymous
// visitors, per client IP.
let key = match &user {
Some(u) => format!("acct:{}", u.id),
None => match client_ip(req.headers(), peer) {
Some(ip) => format!("ip:{ip}"),
None => "anon".to_string(),
},
};
if let Some(state) = req.extensions().get::<Arc<AppState>>() {
if !state.demo_rate_limiter.check(&key) {
return (
StatusCode::TOO_MANY_REQUESTS,
axum::Json(json!({
"error": "rate_limited",
"message": "Too many requests — slow down, or sign in for full access",
})),
)
.into_response();
}
}
next.run(req).await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn counts_filters_in_query() {
assert_eq!(filter_count(""), 0);
assert_eq!(filter_count("resolution=9"), 0);
assert_eq!(filter_count("filters=price%3A1%3A2"), 1);
assert_eq!(
filter_count("filters=price%3A1%3A2%3B%3Bbeds%3A2%3A4&resolution=9"),
2
);
// Trailing/empty entries are ignored.
assert_eq!(filter_count("filters=price%3A1%3A2%3B%3B"), 1);
}
#[test]
fn detects_share_code() {
assert!(!has_share_code(""));
assert!(!has_share_code("filters=a%3B%3Bb"));
assert!(!has_share_code("share=")); // empty value doesn't count
assert!(has_share_code("share=abc123"));
assert!(has_share_code("bounds=1,2,3,4&share=xyz&filters=a"));
}
#[test]
fn rate_limiter_blocks_past_the_window_limit() {
let limiter = DemoRateLimiter::new();
// The first DEMO_RATE_LIMIT_MAX hits for a key pass; the next is blocked.
for _ in 0..DEMO_RATE_LIMIT_MAX {
assert!(limiter.check("acct:abc"));
}
assert!(!limiter.check("acct:abc"));
// A different key has its own independent budget.
assert!(limiter.check("acct:xyz"));
}
}

View file

@ -51,6 +51,7 @@ const LISTING_BOUNDS_EPSILON_DEGREES: f64 = 0.00001;
pub async fn get_actual_listings(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<ActualListingsParams>,
) -> Result<Json<ActualListingsResponse>, Response> {
let state = shared.load_state();
@ -70,7 +71,7 @@ pub async fn get_actual_listings(
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
let quant = state.data.quant_ref();
let poi_quant = state.data.poi_metrics.quant_ref();

View file

@ -14,6 +14,7 @@ use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::NAN_U16;
use crate::data::travel_time::TravelData;
use crate::data::{PostcodePoiMetrics, QuantRef};
use crate::features;
use crate::licensing::{check_license_bounds, resolve_share_code};
@ -369,15 +370,98 @@ fn bounds_for_postcode_indices(
(south, west, north, east)
}
/// A travel-time column derived from the active `travel` / `tt` query params.
/// Each active travel destination becomes one column showing the per-postcode
/// median (or best-case) travel time in minutes.
struct TravelColumn {
/// Human-readable destination name for the header (e.g. "Bank tube station").
header: String,
/// Description-row text (e.g. "Travel time by public transport (minutes)").
description: String,
/// Postcode → travel time data for this destination.
data: TravelData,
/// Whether to report the best-case (5th percentile) time instead of median.
use_best: bool,
}
/// Parse the repeated `tt` frontend-state params into a (mode, slug) → label map.
/// Each value looks like `mode:slug:EncodedLabel[:b][:min:max]`, where the label
/// was percent-encoded with the frontend's `encodeURIComponent`.
fn parse_travel_labels(tt_params: &[String]) -> FxHashMap<(String, String), String> {
let mut map = FxHashMap::default();
for raw in tt_params {
// splitn(4) keeps the label as a single piece; it never contains a raw
// ':' because encodeURIComponent escapes it to %3A.
let mut parts = raw.splitn(4, ':');
let (Some(mode), Some(slug), Some(encoded_label)) =
(parts.next(), parts.next(), parts.next())
else {
continue;
};
let label = urlencoding::decode(encoded_label)
.map(|cow| cow.into_owned())
.unwrap_or_else(|_| encoded_label.to_string());
if !label.is_empty() {
map.insert((mode.to_string(), slug.to_string()), label);
}
}
map
}
/// Turn a destination slug into a display name when no label is supplied.
/// "bank-tube-station" → "Bank tube station".
fn humanize_slug(slug: &str) -> String {
let mut s = slug.replace('-', " ");
if let Some(first) = s.get_mut(0..1) {
first.make_ascii_uppercase();
}
s
}
/// Friendly name for a travel mode (incl. transit variants) for the desc row.
fn pretty_travel_mode(mode: &str) -> &'static str {
match mode {
"car" => "car",
"bicycle" => "bike",
"walking" => "walking",
m if m.starts_with("transit") => "public transport",
_ => "travel",
}
}
/// The reported travel time (median, or best-case when `use_best`) for a
/// postcode, or None when the destination has no data for it.
#[inline]
fn travel_minutes_for(data: &TravelData, postcode: &str, use_best: bool) -> Option<i16> {
data.get(postcode).map(|row| {
if use_best {
row.best_minutes.unwrap_or(row.minutes)
} else {
row.minutes
}
})
}
pub async fn get_export(
State(shared): State<Arc<SharedState>>,
headers: HeaderMap,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
uri: Uri,
Query(params): Query<ExportParams>,
) -> Result<impl IntoResponse, axum::response::Response> {
let state = shared.load_state();
// Exporting requires an account — no anonymous/demo exports.
if user.0.is_none() {
return Err((
StatusCode::UNAUTHORIZED,
[(header::CONTENT_TYPE, "application/json")],
r#"{"error":"account_required","message":"Sign in to export data"}"#,
)
.into_response());
}
// Two modes: bounds-based (default) and explicit postcode list.
let postcode_list = match params.postcodes.as_deref() {
Some(raw) if !raw.trim().is_empty() => {
@ -418,7 +502,7 @@ pub async fn get_export(
}
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
let quant = state.data.quant_ref();
let poi_quant = state.data.poi_metrics.quant_ref();
@ -450,20 +534,17 @@ pub async fn get_export(
} else {
params.filters
};
let travel_entries = if is_postcode_mode {
Vec::new()
} else {
parse_optional_travel(params.travel.as_deref())
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
};
let has_travel_filters = travel_entries
.iter()
.any(|entry| entry.filter_min.is_some() && entry.filter_max.is_some());
let travel_state_params = if is_postcode_mode {
Vec::new()
} else {
collect_travel_state_params(uri.query())
};
// Travel times are per-postcode global data, so they appear as columns in
// both bounds and list mode. The time-range *filter* is only applied on the
// bounds path (`has_travel_filters` below); list mode never filters rows, so
// every requested postcode still shows up with its travel time.
let travel_entries = parse_optional_travel(params.travel.as_deref())
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let has_travel_filters = !is_postcode_mode
&& travel_entries
.iter()
.any(|entry| entry.filter_min.is_some() && entry.filter_max.is_some());
let travel_state_params = collect_travel_state_params(uri.query());
let overlay_state_params = if is_postcode_mode {
Vec::new()
} else {
@ -553,6 +634,33 @@ pub async fn get_export(
let postcode_data = &state.postcode_data;
let poi_metrics = &state.data.poi_metrics;
let travel_data = load_travel_data(&state.travel_time_store, &travel_entries)?;
// One column per active travel destination. `travel_data` is built by
// mapping over `travel_entries` in order, so the two are index-aligned.
// Display labels come from the `tt` frontend-state params.
let travel_labels = parse_travel_labels(&travel_state_params);
let travel_columns: Vec<TravelColumn> = travel_entries
.iter()
.zip(travel_data.iter())
.map(|(entry, data)| {
let header = travel_labels
.get(&(entry.mode.clone(), entry.slug.clone()))
.cloned()
.unwrap_or_else(|| humanize_slug(&entry.slug));
let description = format!(
"Travel time by {}{} (minutes)",
pretty_travel_mode(&entry.mode),
if entry.use_best { ", best case" } else { "" }
);
TravelColumn {
header,
description,
data: Arc::clone(data),
use_best: entry.use_best,
}
})
.collect();
let poi_offset = num_features;
let total_export_features = num_features + poi_metrics.num_features();
let (pc_interner, pc_keys) = state.data.postcode_parts();
@ -701,6 +809,15 @@ pub async fn get_export(
ordered
};
// The "All Data" sheet lists property/area features only. POI amenity
// counts & distances (virtual indices >= poi_offset) are reserved for the
// "Selected" sheet, where they show up when the user filters/selects them.
let all_data_feature_indices: Vec<usize> = all_feature_indices
.iter()
.copied()
.filter(|&idx| idx < poi_offset)
.collect();
// Filter-only feature indices for the Selected sheet
let filter_feature_indices: Vec<usize> = filter_feature_names
.iter()
@ -839,6 +956,12 @@ pub async fn get_export(
let group_label_fmt = Format::new().set_bold().set_font_color("#1F4E79");
let group_count_fmt = Format::new().set_bold();
// Travel-time cells render the integer minute count with a " min" suffix.
let travel_num_fmt = Format::new().set_num_format("0\" min\"");
// Postcode-centroid coordinates (WGS84), ~1 m precision at 5 decimals.
let coord_num_fmt = Format::new().set_num_format("0.00000");
// Dashboard URL
let dashboard_url = format!(
"{}/dashboard?{}",
@ -847,14 +970,16 @@ pub async fn get_export(
);
// Two sheets in both modes: "Selected" (just the features behind the
// active filters) and "All Data" (every feature). The Selected sheet
// carries the dashboard link + screenshot only in bounds mode, where the
// export is tied to a map view; in list mode the user picked the
// postcodes explicitly, so there's no map view to link or screenshot.
// active filters, including any POI amenity metrics) and "All Data"
// (every property/area feature, but no POI amenity counts/distances —
// those live only on the Selected sheet). The Selected sheet carries the
// dashboard link + screenshot only in bounds mode, where the export is
// tied to a map view; in list mode the user picked the postcodes
// explicitly, so there's no map view to link or screenshot.
let is_list_mode = postcode_list_entries.is_some();
let sheet_configs: Vec<(&str, &[usize], bool)> = vec![
("Selected", &filter_feature_indices, !is_list_mode),
("All Data", &all_feature_indices, false),
("All Data", &all_data_feature_indices, false),
];
for (sheet_name, feat_indices, include_header) in &sheet_configs {
@ -904,6 +1029,10 @@ pub async fn get_export(
// Header row
let header_row = current_row;
// Travel-time columns sit to the right of the feature columns, with
// the Latitude/Longitude pair last.
let travel_col_base = (feat_indices.len() + 2) as u16;
let coord_col_base = travel_col_base + travel_columns.len() as u16;
sheet
.write_string_with_format(header_row, 0, "Postcode", &header_fmt)
.map_err(|e| format!("Failed to write header: {e}"))?;
@ -923,6 +1052,24 @@ pub async fn get_export(
.map_err(|e| format!("Failed to write header: {e}"))?;
}
for (i, tc) in travel_columns.iter().enumerate() {
sheet
.write_string_with_format(
header_row,
travel_col_base + i as u16,
&tc.header,
&header_fmt,
)
.map_err(|e| format!("Failed to write travel header: {e}"))?;
}
sheet
.write_string_with_format(header_row, coord_col_base, "Latitude", &header_fmt)
.map_err(|e| format!("Failed to write header: {e}"))?;
sheet
.write_string_with_format(header_row, coord_col_base + 1, "Longitude", &header_fmt)
.map_err(|e| format!("Failed to write header: {e}"))?;
// Description row
let desc_row = header_row + 1;
sheet
@ -943,6 +1090,29 @@ pub async fn get_export(
.map_err(|e| format!("Failed to write desc: {e}"))?;
}
for (i, tc) in travel_columns.iter().enumerate() {
sheet
.write_string_with_format(
desc_row,
travel_col_base + i as u16,
&tc.description,
&desc_fmt,
)
.map_err(|e| format!("Failed to write travel desc: {e}"))?;
}
sheet
.write_string_with_format(desc_row, coord_col_base, "Postcode centroid", &desc_fmt)
.map_err(|e| format!("Failed to write desc: {e}"))?;
sheet
.write_string_with_format(
desc_row,
coord_col_base + 1,
"Postcode centroid",
&desc_fmt,
)
.map_err(|e| format!("Failed to write desc: {e}"))?;
// Put the collapse/expand controls above each group so the bold
// outcode summary row acts as the header for its postcodes.
sheet.group_symbols_above(true);
@ -976,6 +1146,63 @@ pub async fn get_export(
&integer_feature_indices,
&feat_num_fmts,
)?;
// Travel time rolled up across the group's postcodes, weighted by
// property count to match the property-weighted feature means.
for (i, tc) in travel_columns.iter().enumerate() {
let mut weighted_sum = 0.0_f64;
let mut weight = 0u32;
for &member in &group.members {
let (member_pc_idx, member_agg) = &postcode_aggs[member];
let pc = &postcode_data.postcodes[*member_pc_idx];
if let Some(minutes) = travel_minutes_for(&tc.data, pc, tc.use_best) {
weighted_sum += minutes as f64 * member_agg.count as f64;
weight += member_agg.count;
}
}
if weight > 0 {
let mean = (weighted_sum / weight as f64).round();
sheet
.write_number_with_format(
summary_row,
travel_col_base + i as u16,
mean,
&travel_num_fmt,
)
.map_err(|e| format!("Failed to write travel summary: {e}"))?;
}
}
// Property-weighted centroid of the group's postcodes.
{
let mut lat_sum = 0.0_f64;
let mut lon_sum = 0.0_f64;
let mut weight = 0u32;
for &member in &group.members {
let (member_pc_idx, member_agg) = &postcode_aggs[member];
let (lat, lon) = postcode_data.centroids[*member_pc_idx];
lat_sum += lat as f64 * member_agg.count as f64;
lon_sum += lon as f64 * member_agg.count as f64;
weight += member_agg.count;
}
if weight > 0 {
let w = weight as f64;
sheet
.write_number_with_format(
summary_row,
coord_col_base,
lat_sum / w,
&coord_num_fmt,
)
.map_err(|e| format!("Failed to write latitude: {e}"))?;
sheet
.write_number_with_format(
summary_row,
coord_col_base + 1,
lon_sum / w,
&coord_num_fmt,
)
.map_err(|e| format!("Failed to write longitude: {e}"))?;
}
}
row += 1;
// Individual postcode rows for this outcode.
@ -999,6 +1226,34 @@ pub async fn get_export(
&integer_feature_indices,
&feat_num_fmts,
)?;
for (i, tc) in travel_columns.iter().enumerate() {
if let Some(minutes) = travel_minutes_for(
&tc.data,
&postcode_data.postcodes[*pc_idx],
tc.use_best,
) {
sheet
.write_number_with_format(
row,
travel_col_base + i as u16,
minutes as f64,
&travel_num_fmt,
)
.map_err(|e| format!("Failed to write travel time: {e}"))?;
}
}
let (lat, lon) = postcode_data.centroids[*pc_idx];
sheet
.write_number_with_format(row, coord_col_base, lat as f64, &coord_num_fmt)
.map_err(|e| format!("Failed to write latitude: {e}"))?;
sheet
.write_number_with_format(
row,
coord_col_base + 1,
lon as f64,
&coord_num_fmt,
)
.map_err(|e| format!("Failed to write longitude: {e}"))?;
row += 1;
}
@ -1012,7 +1267,8 @@ pub async fn get_export(
// Sample note
if was_sampled {
let note_row = row + 1;
let total_cols = (feat_indices.len() + 2) as u16;
// +2 for the Latitude/Longitude columns.
let total_cols = (feat_indices.len() + 2 + travel_columns.len() + 2) as u16;
sheet
.merge_range(
note_row,
@ -1043,6 +1299,18 @@ pub async fn get_export(
.set_column_width(col, width)
.map_err(|e| format!("Failed to set column width: {e}"))?;
}
for (i, tc) in travel_columns.iter().enumerate() {
let width = (tc.header.chars().count() as f64 * 1.1).clamp(10.0, 30.0);
sheet
.set_column_width(travel_col_base + i as u16, width)
.map_err(|e| format!("Failed to set travel column width: {e}"))?;
}
sheet
.set_column_width(coord_col_base, 11)
.map_err(|e| format!("Failed to set coordinate column width: {e}"))?;
sheet
.set_column_width(coord_col_base + 1, 11)
.map_err(|e| format!("Failed to set coordinate column width: {e}"))?;
}
let buf = workbook
@ -1128,6 +1396,50 @@ mod tests {
assert_eq!(outcode_of("E14"), "E14");
}
#[test]
fn parse_travel_labels_decodes_label_keyed_by_mode_and_slug() {
// As produced by collect_travel_state_params (outer URL layer already
// decoded); the label is still encodeURIComponent-encoded.
let params = vec![
"transit:bank-tube-station:Bank%20tube%20station:0:52".to_string(),
"car:heathrow-airport:Heathrow%20Airport:b:0:90".to_string(),
];
let labels = parse_travel_labels(&params);
assert_eq!(
labels.get(&("transit".to_string(), "bank-tube-station".to_string())),
Some(&"Bank tube station".to_string())
);
assert_eq!(
labels.get(&("car".to_string(), "heathrow-airport".to_string())),
Some(&"Heathrow Airport".to_string())
);
}
#[test]
fn parse_travel_labels_skips_entries_without_a_label() {
// Empty label (encodeURIComponent("")) and a too-short value are ignored.
let params = vec![
"transit:bank-tube-station::b:0:52".to_string(),
"car:somewhere".to_string(),
];
assert!(parse_travel_labels(&params).is_empty());
}
#[test]
fn humanize_slug_falls_back_to_a_readable_name() {
assert_eq!(humanize_slug("bank-tube-station"), "Bank tube station");
assert_eq!(humanize_slug("london"), "London");
}
#[test]
fn pretty_travel_mode_maps_transit_variants() {
assert_eq!(pretty_travel_mode("transit"), "public transport");
assert_eq!(pretty_travel_mode("transit-no-buses"), "public transport");
assert_eq!(pretty_travel_mode("car"), "car");
assert_eq!(pretty_travel_mode("bicycle"), "bike");
assert_eq!(pretty_travel_mode("walking"), "walking");
}
#[test]
fn export_query_deserializes_when_tt_is_a_single_string() {
let uri: Uri = "/api/export?bounds=1,2,3,4&tt=transit%3Abank%3ABank%2520station%3A0%3A52"

View file

@ -33,6 +33,7 @@ pub struct FilterCountsResponse {
pub async fn get_filter_counts(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<FilterCountsParams>,
) -> Result<Json<FilterCountsResponse>, axum::response::Response> {
let state = shared.load_state();
@ -40,7 +41,7 @@ pub async fn get_filter_counts(
let (south, west, north, east) =
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
let quant = state.data.quant_ref();
let poi_quant = state.data.poi_metrics.quant_ref();

View file

@ -436,6 +436,7 @@ pub(super) fn top_filter_exclusions(
pub async fn get_hexagon_stats(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<HexagonStatsParams>,
) -> Result<Json<HexagonStatsResponse>, axum::response::Response> {
let state = shared.load_state();
@ -455,7 +456,7 @@ pub async fn get_hexagon_stats(
// License check using H3 cell bounds
let h3_bounds = h3_cell_bounds(cell, 0.0);
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, h3_bounds, share_bounds)?;
check_license_bounds(&user.0, h3_bounds, geo.free_zone, share_bounds)?;
let h3_str = params.h3;
let quant = state.data.quant_ref();

View file

@ -252,6 +252,7 @@ fn build_feature_maps(
pub async fn get_hexagons(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<HexagonParams>,
) -> Result<Json<HexagonsResponse>, axum::response::Response> {
let state = shared.load_state();
@ -262,7 +263,7 @@ pub async fn get_hexagons(
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
let quant = state.data.quant_ref();
let poi_quant = state.data.poi_metrics.quant_ref();

View file

@ -57,6 +57,7 @@ fn destination_coordinates(place_data: &PlaceData, slug: &str) -> Option<(f32, f
pub async fn get_journey(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
query: axum::extract::Query<JourneyQuery>,
) -> Result<Json<JourneyResponse>, axum::response::Response> {
let state = shared.load_state();
@ -72,7 +73,7 @@ pub async fn get_journey(
let (lat, lon) = state.postcode_data.centroids[pc_idx];
let share_bounds = resolve_share_code(&state, query.share.as_deref()).await;
check_license_point(&user.0, lat as f64, lon as f64, share_bounds)?;
check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
if !store.has_destination(&query.mode, &query.slug) {
return Err((

View file

@ -41,6 +41,7 @@ pub struct PostcodePropertiesParams {
pub async fn get_postcode_properties(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<PostcodePropertiesParams>,
) -> Result<Json<PropertyListResponse>, axum::response::Response> {
let state = shared.load_state();
@ -64,6 +65,7 @@ pub async fn get_postcode_properties(
&user.0,
centroid_lat as f64,
centroid_lon as f64,
geo.free_zone,
share_bounds,
)?;

View file

@ -15,7 +15,7 @@ use crate::state::SharedState;
use crate::utils::normalize_postcode;
use super::hexagon_stats::{
parse_area_stats_field_set, top_filter_exclusions, HexagonStatsResponse,
parse_area_stats_field_set, top_filter_exclusions, EnumFeatureStats, HexagonStatsResponse,
};
use super::stats;
use super::travel_time::{load_travel_data, parse_optional_travel, row_passes_travel_filters};
@ -38,6 +38,7 @@ pub struct PostcodeStatsParams {
pub async fn get_postcode_stats(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<PostcodeStatsParams>,
) -> Result<Json<HexagonStatsResponse>, axum::response::Response> {
let state = shared.load_state();
@ -63,6 +64,7 @@ pub async fn get_postcode_stats(
&user.0,
centroid_lat as f64,
centroid_lon as f64,
geo.free_zone,
share_bounds,
)?;
@ -157,7 +159,7 @@ pub async fn get_postcode_stats(
&field_set,
);
let (mut numeric_features, enum_features_out) = stats::compute_feature_stats(
let (mut numeric_features, mut enum_features_out) = stats::compute_feature_stats(
&matching_rows,
&state.data,
&state.data.feature_names,
@ -173,6 +175,31 @@ pub async fn get_postcode_stats(
&field_set,
));
// The council-house count is an attribute of the postcode itself, not of
// the currently filter-matching subset: recompute it over every property
// in the postcode so toggling filters never changes it. (When no filters
// are applied, `area_rows == matching_rows` and this is a no-op.)
if let Some(&council_idx) = state
.feature_name_to_index
.get(crate::consts::FORMER_COUNCIL_HOUSE_FEATURE)
{
enum_features_out.retain(|f| f.name != crate::consts::FORMER_COUNCIL_HOUSE_FEATURE);
let included = !fields_specified
|| field_set.contains(crate::consts::FORMER_COUNCIL_HOUSE_FEATURE);
if included {
if let Some(counts) =
stats::compute_enum_feature_counts(&area_rows, &state.data, council_idx)
{
if !counts.is_empty() {
enum_features_out.push(EnumFeatureStats {
name: crate::consts::FORMER_COUNCIL_HOUSE_FEATURE.to_string(),
counts,
});
}
}
}
}
let elapsed = start_time.elapsed();
info!(
postcode = %postcode_str,

View file

@ -54,6 +54,7 @@ pub struct PostcodeParams {
pub async fn get_postcodes(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<PostcodeParams>,
) -> Result<Json<PostcodesResponse>, axum::response::Response> {
let state = shared.load_state();
@ -61,7 +62,7 @@ pub async fn get_postcodes(
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
let quant = state.data.quant_ref();
let poi_quant = state.data.poi_metrics.quant_ref();

View file

@ -11,7 +11,7 @@ use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::PROPERTIES_LIMIT;
use crate::data::{HistoricalPrice, RenovationEvent};
use crate::data::{HistoricalPrice, RenovationEvent, TenureEvent};
use crate::licensing::{check_license_bounds, resolve_share_code};
use crate::parsing::{
cell_for_row_cached, h3_cell_bounds, needs_parent, parse_field_indices_with_poi,
@ -67,6 +67,9 @@ pub struct Property {
#[serde(skip_serializing_if = "Vec::is_empty")]
pub historical_prices: Vec<HistoricalPrice>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub tenure_history: Vec<TenureEvent>,
#[serde(flatten)]
pub features: FxHashMap<String, f32>,
}
@ -240,6 +243,7 @@ pub fn build_property(
lon: state.data.lon[row],
renovation_history: state.data.renovation_history(row).to_vec(),
historical_prices: state.data.historical_prices(row).to_vec(),
tenure_history: state.data.tenure_history(row).to_vec(),
property_sub_type: state.data.property_sub_type(row).map(String::from),
price_qualifier: state.data.price_qualifier(row).map(String::from),
former_council_house: lookup_enum_value(
@ -270,6 +274,7 @@ pub fn build_property(
pub async fn get_hexagon_properties(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<HexagonPropertiesParams>,
) -> Result<Json<PropertyListResponse>, axum::response::Response> {
let state = shared.load_state();
@ -289,7 +294,7 @@ pub async fn get_hexagon_properties(
// License check using H3 cell bounds
let h3_bounds = h3_cell_bounds(cell, 0.0);
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
check_license_bounds(&user.0, h3_bounds, share_bounds)?;
check_license_bounds(&user.0, h3_bounds, geo.free_zone, share_bounds)?;
let h3_str = params.h3;
let quant = state.data.quant_ref();

View file

@ -255,6 +255,36 @@ pub fn compute_feature_stats(
(numeric_features, enum_features_out)
}
/// Count occurrences of each variant of a single enum feature across `rows`.
///
/// Unlike [`compute_feature_stats`], which is driven by the filter-matching
/// subset, this lets a caller compute a count that should reflect a whole area
/// regardless of the active filters (e.g. the council-house count, which is an
/// attribute of the postcode itself, not of the currently filtered properties).
/// Returns `None` if `feature_idx` is not an enum feature.
pub fn compute_enum_feature_counts(
rows: &[usize],
data: &PropertyData,
feature_idx: usize,
) -> Option<HashMap<String, u64>> {
let variants = data.enum_values.get(&feature_idx)?;
let mut value_counts = vec![0u64; variants.len()];
for &row in rows {
let value = data.get_feature(row, feature_idx);
if value.is_finite() && value >= 0.0 && (value as usize) < value_counts.len() {
value_counts[value as usize] += 1;
}
}
Some(
value_counts
.iter()
.enumerate()
.filter(|(_, &count)| count > 0)
.map(|(idx, &count)| (variants[idx].clone(), count))
.collect(),
)
}
/// Compute property-weighted per-year crime means across the selection.
///
/// Each matching property contributes its postcode's per-year counts (incidents
@ -447,3 +477,42 @@ pub fn compute_poi_feature_stats(
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::consts::NAN_U16;
fn enum_data(values: &[u16]) -> PropertyData {
let mut data = PropertyData::empty_for_tests();
data.num_features = 1;
data.num_numeric = 0; // single enum feature at index 0
data.feature_data = values.to_vec();
data.enum_values
.insert(0, vec!["Yes".to_string(), "No".to_string()]);
data
}
#[test]
fn enum_counts_tally_only_given_rows() {
// Rows: Yes, No, Yes, <missing>
let data = enum_data(&[0, 1, 0, NAN_U16]);
let all = compute_enum_feature_counts(&[0, 1, 2, 3], &data, 0).unwrap();
assert_eq!(all.get("Yes"), Some(&2));
assert_eq!(all.get("No"), Some(&1));
// A filter-matching subset would yield a different tally — confirming
// the count is driven purely by the rows passed in (so callers can pass
// the full area to make it filter-independent).
let subset = compute_enum_feature_counts(&[0, 3], &data, 0).unwrap();
assert_eq!(subset.get("Yes"), Some(&1));
assert_eq!(subset.get("No"), None);
}
#[test]
fn enum_counts_none_for_non_enum_feature() {
let data = enum_data(&[0, 1]);
assert!(compute_enum_feature_counts(&[0, 1], &data, 1).is_none());
}
}

View file

@ -17,7 +17,7 @@ use crate::utils::GridIndex;
pub struct AppState {
pub data: PropertyData,
pub grid: GridIndex,
/// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (12).
/// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (9).
/// Parent cells for lower resolutions derived via CellIndex::parent().
pub h3_cells: Vec<u64>,
/// O(1) lookup: feature name → index in feature_names/feature_data
@ -85,6 +85,8 @@ pub struct AppState {
pub stripe_referral_coupon_id: String,
/// Bugsink/Sentry-compatible browser error reporting config injected into served HTML.
pub bugsink_frontend_config: Option<BugsinkFrontendConfig>,
/// Per-key rate limiter for unlicensed ("demo") API traffic.
pub demo_rate_limiter: Arc<crate::ratelimit::DemoRateLimiter>,
}
#[cfg(test)]
@ -182,6 +184,7 @@ impl AppState {
stripe_webhook_secret: "whsec_test_secret".to_string(),
stripe_referral_coupon_id: "couponTest30".to_string(),
bugsink_frontend_config: None,
demo_rate_limiter: Arc::new(crate::ratelimit::DemoRateLimiter::new()),
}
}
}