new demo mode & tenure
This commit is contained in:
parent
7656f24544
commit
4a0f00f2a4
64 changed files with 2875 additions and 338 deletions
|
|
@ -3,9 +3,9 @@ pub const QUANT_SCALE: f32 = 65534.0;
|
|||
|
||||
pub const HISTOGRAM_BINS: usize = 100;
|
||||
|
||||
pub const H3_PRECOMPUTE_MAX: u8 = 12;
|
||||
pub const H3_PRECOMPUTE_MAX: u8 = 9;
|
||||
pub const H3_REQUEST_MIN: u8 = 4;
|
||||
pub const H3_REQUEST_MAX: u8 = 12;
|
||||
pub const H3_REQUEST_MAX: u8 = 9;
|
||||
|
||||
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
|
||||
|
||||
|
|
@ -15,6 +15,12 @@ pub const MAX_POIS_PER_REQUEST: usize = 3000;
|
|||
|
||||
pub const PROPERTIES_LIMIT: usize = 100;
|
||||
pub const PLACES_LIMIT: usize = 20;
|
||||
|
||||
/// Enum feature whose count is an attribute of the postcode/area as a whole
|
||||
/// rather than of the filter-matching subset: it is always computed over every
|
||||
/// property in the area so applying filters never changes it. Must match the
|
||||
/// feature name in `features.rs`.
|
||||
pub const FORMER_COUNCIL_HOUSE_FEATURE: &str = "Former council house";
|
||||
pub const PRICE_HISTORY_POINTS_LIMIT: usize = 5000;
|
||||
pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02;
|
||||
|
||||
|
|
@ -26,9 +32,29 @@ pub const AI_FILTERS_WEEKLY_TOKEN_LIMIT: u64 = 10_000_000;
|
|||
/// Timeout for outbound HTTP service calls (seconds).
|
||||
pub const SERVICE_CALL_TIMEOUT: u64 = 120;
|
||||
|
||||
/// Demo free zone bounds (south, west, north, east) — inner London, roughly zone 1.
|
||||
/// Users without a license can only query data within these bounds.
|
||||
pub const FREE_ZONE_BOUNDS: (f64, f64, f64, f64) = (51.44, -0.31, 51.59, 0.05);
|
||||
/// Half-spans of the demo free zone box, in degrees. The box is built this wide
|
||||
/// around the visitor's approximate (IP-derived) location — or the fallback
|
||||
/// centre below when the IP can't be geolocated. Sized to match the original
|
||||
/// inner-London "zone 1" box (≈0.15° lat × 0.36° lon).
|
||||
pub const FREE_ZONE_HALF_LAT: f64 = 0.075;
|
||||
pub const FREE_ZONE_HALF_LON: f64 = 0.18;
|
||||
|
||||
/// Fallback demo free-zone centre (Canary Wharf), used when the visitor hasn't
|
||||
/// supplied a demo centre — i.e. they declined the "check your area" prompt or no
|
||||
/// GPS fix was available. When they consent, the browser sends their own centre.
|
||||
pub const FREE_ZONE_FALLBACK_LAT: f64 = 51.5054;
|
||||
pub const FREE_ZONE_FALLBACK_LON: f64 = -0.0235;
|
||||
|
||||
/// Demo (unlicensed) users may apply at most this many filters. Enforced both
|
||||
/// client-side (UX) and server-side (anti-DoS); must match the frontend constant.
|
||||
pub const DEMO_MAX_FILTERS: usize = 5;
|
||||
/// Sliding-window rate limit for unlicensed traffic to `/api/` endpoints, keyed by
|
||||
/// account id (preferred) or client IP. Generous enough for active demo use, tight
|
||||
/// enough to blunt tile-by-tile scraping. Licensed users/admins are exempt.
|
||||
pub const DEMO_RATE_LIMIT_MAX: usize = 600;
|
||||
pub const DEMO_RATE_LIMIT_WINDOW_SECS: u64 = 60;
|
||||
/// Soft cap on tracked rate-limit keys before stale ones are pruned.
|
||||
pub const DEMO_RATE_LIMIT_MAX_KEYS: usize = 50_000;
|
||||
|
||||
pub const SHARE_CACHE_TTL_SECS: u64 = 300;
|
||||
pub const SHARE_CACHE_MAX_ENTRIES: usize = 1024;
|
||||
|
|
|
|||
|
|
@ -67,6 +67,6 @@ pub use poi::{resolve_poi_category_filter, POICategoryGroup, POIData, SchoolMeta
|
|||
pub use postcodes::{OutcodeData, PostcodeData};
|
||||
pub use property::{
|
||||
precompute_h3, FeatureStats, Histogram, HistoricalPrice, PostcodePoiMetrics, PropertyData,
|
||||
QuantRef, RenovationEvent,
|
||||
QuantRef, RenovationEvent, TenureEvent,
|
||||
};
|
||||
pub use travel_time::{slugify, TravelTimeStore};
|
||||
|
|
|
|||
|
|
@ -251,6 +251,26 @@ pub(super) fn address_search_tokens(text: &str) -> Vec<String> {
|
|||
tokens
|
||||
}
|
||||
|
||||
/// Search tokens for a property row, covering its display address plus an
|
||||
/// alternative spelling (`alt`, the EPC form) when distinct — so the property is
|
||||
/// findable by either form. The result is deduped: each token appears at most
|
||||
/// once, which the inverted-index posting lists rely on (a token must post a
|
||||
/// given row exactly once to keep those lists strictly ascending and unique).
|
||||
pub(super) fn merged_address_search_tokens(display: &str, alt: Option<&str>) -> Vec<String> {
|
||||
let mut tokens = address_search_tokens(display);
|
||||
if let Some(alt) = alt {
|
||||
let alt = alt.trim();
|
||||
if !alt.is_empty() && alt != display.trim() {
|
||||
for token in address_search_tokens(alt) {
|
||||
if !tokens.contains(&token) {
|
||||
tokens.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
fn is_address_search_token(token: &str) -> bool {
|
||||
if looks_like_postcode_fragment(token) {
|
||||
return false;
|
||||
|
|
@ -929,6 +949,60 @@ mod tests {
|
|||
assert_eq!(parsed.text_groups[1].alternatives, vec!["cour".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_tokens_index_both_address_forms() {
|
||||
// A property whose price-paid form is "10 Park Road" but whose EPC form
|
||||
// names the building ("Kingswood House 10 Park Road") must be findable by
|
||||
// the building name too: the distinctive "kingswood" token comes only
|
||||
// from the EPC form.
|
||||
let display_only = address_search_tokens("10 Park Road");
|
||||
assert!(!display_only.contains(&"kingswood".to_string()));
|
||||
|
||||
let merged =
|
||||
merged_address_search_tokens("10 Park Road", Some("Kingswood House 10 Park Road"));
|
||||
assert!(merged.contains(&"kingswood".to_string()));
|
||||
// Tokens shared by both forms are still present and not duplicated.
|
||||
assert!(merged.contains(&"park".to_string()));
|
||||
assert!(merged.contains(&"road".to_string()));
|
||||
assert!(merged.contains(&"10".to_string()));
|
||||
|
||||
// No token repeats — posting-list uniqueness depends on this.
|
||||
let mut deduped = merged.clone();
|
||||
deduped.sort_unstable();
|
||||
deduped.dedup();
|
||||
assert_eq!(deduped.len(), merged.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_tokens_ignore_redundant_or_missing_alt_form() {
|
||||
let baseline = address_search_tokens("12 Baker Street");
|
||||
// An EPC form identical to the display adds nothing.
|
||||
assert_eq!(
|
||||
merged_address_search_tokens("12 Baker Street", Some("12 Baker Street")),
|
||||
baseline
|
||||
);
|
||||
// A blank EPC form (whitespace) and an absent one are both no-ops.
|
||||
assert_eq!(
|
||||
merged_address_search_tokens("12 Baker Street", Some(" ")),
|
||||
baseline
|
||||
);
|
||||
assert_eq!(
|
||||
merged_address_search_tokens("12 Baker Street", None),
|
||||
baseline
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_tokens_handle_epc_only_display() {
|
||||
// EPC-only dwellings have no price-paid form, so the EPC address IS the
|
||||
// display; passing it as both arguments must not duplicate its tokens.
|
||||
let epc = "Flat 4 Rosewood Court";
|
||||
assert_eq!(
|
||||
merged_address_search_tokens(epc, Some(epc)),
|
||||
address_search_tokens(epc)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_search_tokens_keep_actual_address_terms_for_scoring() {
|
||||
let tokens = address_search_tokens("Flat 2, 10 Downing Cour");
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ use crate::consts::{NAN_U16, QUANT_SCALE};
|
|||
use crate::features::{self, Bounds};
|
||||
|
||||
use super::address_search::{
|
||||
address_search_tokens, build_address_prefix_index, is_address_candidate_token,
|
||||
build_address_prefix_index, is_address_candidate_token, merged_address_search_tokens,
|
||||
};
|
||||
use super::poi_metrics::{PostcodePoiMetrics, NO_POI_METRIC_ROW};
|
||||
use super::stats::{column_to_f32_vec, compute_feature_stats, FeatureStats, Histogram};
|
||||
use super::{HistoricalPrice, PropertyData, RenovationEvent};
|
||||
use super::{HistoricalPrice, PropertyData, RenovationEvent, TenureEvent};
|
||||
|
||||
const MISSING_COORDINATE_SAMPLE_LIMIT: usize = 10;
|
||||
const COUNTRY_COLUMN_CANDIDATES: &[&str] = &[
|
||||
|
|
@ -330,6 +330,10 @@ impl PropertyData {
|
|||
if has_historical_prices {
|
||||
select_exprs.push(col("historical_prices"));
|
||||
}
|
||||
let has_tenure_history = schema.get("tenure_history").is_some();
|
||||
if has_tenure_history {
|
||||
select_exprs.push(col("tenure_history"));
|
||||
}
|
||||
let df = combined_lf
|
||||
.filter(col("lat").is_not_null().and(col("lon").is_not_null()))
|
||||
.select(select_exprs)
|
||||
|
|
@ -439,7 +443,6 @@ impl PropertyData {
|
|||
.collect()
|
||||
};
|
||||
|
||||
let address_raw = extract_string_col(&df, "Address per Property Register")?;
|
||||
let postcode_raw = extract_string_col(&df, "Postcode")?;
|
||||
|
||||
// Extract optional string columns
|
||||
|
|
@ -468,6 +471,13 @@ impl PropertyData {
|
|||
};
|
||||
let property_sub_type_raw = extract_optional_string_col(&df, "Property sub-type")?;
|
||||
let price_qualifier_raw = extract_optional_string_col(&df, "Price qualifier")?;
|
||||
// Display + search addresses. The price-paid form ("Address per Property
|
||||
// Register") is the primary display address; the EPC form ("Address per
|
||||
// EPC") is an alternative spelling indexed alongside it so a property is
|
||||
// findable by either. For EPC-only dwellings (no Land Registry sale) the
|
||||
// price-paid form is null, so the EPC form becomes the display address.
|
||||
let pp_address_raw = extract_optional_string_col(&df, "Address per Property Register")?;
|
||||
let epc_address_raw = extract_optional_string_col(&df, "Address per EPC")?;
|
||||
|
||||
tracing::info!("Building enum features");
|
||||
// enum_col_major: Vec<(values_list, encoded_as_f32)>
|
||||
|
|
@ -675,6 +685,57 @@ impl PropertyData {
|
|||
FxHashMap::default()
|
||||
};
|
||||
|
||||
// Extract tenure_history: List<Struct{year: i32, status: str}>
|
||||
let mut tenure_raw: FxHashMap<u32, Vec<TenureEvent>> = if has_tenure_history {
|
||||
tracing::info!("Extracting tenure history");
|
||||
let tenure_col = df
|
||||
.column("tenure_history")
|
||||
.context("Missing tenure_history column")?;
|
||||
let list_ca = tenure_col
|
||||
.list()
|
||||
.context("tenure_history is not a list column")?;
|
||||
|
||||
let mut history: FxHashMap<u32, Vec<TenureEvent>> = FxHashMap::default();
|
||||
for old_row in 0..row_count {
|
||||
if let Some(inner) = list_ca.get_as_series(old_row) {
|
||||
if inner.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let structs = inner
|
||||
.struct_()
|
||||
.context("tenure_history inner is not a struct")?;
|
||||
let years = structs
|
||||
.field_by_name("year")
|
||||
.context("Missing 'year' field in tenure_history struct")?;
|
||||
let statuses = structs
|
||||
.field_by_name("status")
|
||||
.context("Missing 'status' field in tenure_history struct")?;
|
||||
|
||||
let mut row_events = Vec::new();
|
||||
for idx in 0..inner.len() {
|
||||
let year = years.get(idx).context("Failed to get year value")?;
|
||||
let status = statuses.get(idx).context("Failed to get status value")?;
|
||||
if let (AnyValue::Int32(yr), AnyValue::String(st)) = (&year, &status) {
|
||||
row_events.push(TenureEvent {
|
||||
year: *yr,
|
||||
status: st.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if !row_events.is_empty() {
|
||||
history.insert(old_row as u32, row_events);
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!(
|
||||
properties_with_tenure_changes = history.len(),
|
||||
"Tenure history extracted"
|
||||
);
|
||||
history
|
||||
} else {
|
||||
FxHashMap::default()
|
||||
};
|
||||
|
||||
// Free the projected joined frame before building the row-major matrix.
|
||||
drop(df);
|
||||
|
||||
|
|
@ -712,9 +773,21 @@ impl PropertyData {
|
|||
})
|
||||
.context("Required numeric column 'Last known price' not configured")?;
|
||||
|
||||
// Build contiguous address buffer and address search index (permuted)
|
||||
// Build contiguous address buffer and address search index (permuted).
|
||||
// Each row's posting lists cover BOTH address spellings we hold — the
|
||||
// price-paid form and the EPC form — so a property is findable by either;
|
||||
// the display address prefers the price-paid form, falling back to the EPC
|
||||
// form for never-sold (EPC-only) dwellings whose price-paid form is null.
|
||||
tracing::info!("Building interned strings");
|
||||
let total_addr_bytes: usize = address_raw.iter().map(|text| text.len()).sum();
|
||||
// Display address for a row, preferring the price-paid form and falling
|
||||
// back to the EPC form (so EPC-only dwellings still have a display string).
|
||||
let coalesced_display = |old_row: usize| -> &str {
|
||||
match pp_address_raw[old_row].as_deref() {
|
||||
Some(pp) if !pp.is_empty() => pp,
|
||||
_ => epc_address_raw[old_row].as_deref().unwrap_or(""),
|
||||
}
|
||||
};
|
||||
let total_addr_bytes: usize = (0..row_count).map(|row| coalesced_display(row).len()).sum();
|
||||
let mut address_buffer = String::with_capacity(total_addr_bytes);
|
||||
let mut address_offsets = Vec::with_capacity(row_count);
|
||||
let mut address_lengths = Vec::with_capacity(row_count);
|
||||
|
|
@ -724,14 +797,19 @@ impl PropertyData {
|
|||
let mut address_search_token_offsets = Vec::with_capacity(row_count);
|
||||
let mut address_search_token_lengths = Vec::with_capacity(row_count);
|
||||
for (new_row, &perm_index) in perm.iter().enumerate() {
|
||||
let addr = &address_raw[perm_index as usize];
|
||||
let old_row = perm_index as usize;
|
||||
let display = coalesced_display(old_row);
|
||||
let offset = address_buffer.len() as u32;
|
||||
let length = addr.len().min(u16::MAX as usize) as u16;
|
||||
let length = display.len().min(u16::MAX as usize) as u16;
|
||||
address_offsets.push(offset);
|
||||
address_lengths.push(length);
|
||||
address_buffer.push_str(&addr[..length as usize]);
|
||||
address_buffer.push_str(&display[..length as usize]);
|
||||
|
||||
let search_tokens = address_search_tokens(addr);
|
||||
// Tokens cover the display address plus the EPC-form spelling when
|
||||
// distinct, so the property is findable by either form (deduped so
|
||||
// each token posts this row exactly once).
|
||||
let search_tokens =
|
||||
merged_address_search_tokens(display, epc_address_raw[old_row].as_deref());
|
||||
let token_offset = address_search_token_keys.len() as u32;
|
||||
let token_length = search_tokens.len().min(u16::MAX as usize) as u16;
|
||||
address_search_token_offsets.push(token_offset);
|
||||
|
|
@ -834,6 +912,18 @@ impl PropertyData {
|
|||
map
|
||||
};
|
||||
|
||||
// Re-key tenure_history by permuted row index
|
||||
let tenure_history: FxHashMap<u32, Vec<TenureEvent>> = {
|
||||
let mut map =
|
||||
FxHashMap::with_capacity_and_hasher(tenure_raw.len(), Default::default());
|
||||
for (new_row, &old_row) in perm.iter().enumerate() {
|
||||
if let Some(events) = tenure_raw.remove(&old_row) {
|
||||
map.insert(new_row as u32, events);
|
||||
}
|
||||
}
|
||||
map
|
||||
};
|
||||
|
||||
// Permute optional string columns into sparse HashMaps
|
||||
let property_sub_type: FxHashMap<u32, String> = {
|
||||
let mut map = FxHashMap::default();
|
||||
|
|
@ -968,6 +1058,7 @@ impl PropertyData {
|
|||
approx_build_date_bits,
|
||||
renovation_history,
|
||||
historical_prices,
|
||||
tenure_history,
|
||||
property_sub_type,
|
||||
price_qualifier,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -40,6 +40,16 @@ pub struct HistoricalPrice {
|
|||
pub price: i64,
|
||||
}
|
||||
|
||||
/// A point on the property's tenure timeline: the year a certificate first
|
||||
/// recorded a new occupancy `status` ("Owner-occupied" / "Rented (private)" /
|
||||
/// "Rented (social)"). Derived per-EPC and reduced to change points by the
|
||||
/// pipeline, so this surfaces *when* a home was let out vs lived in.
|
||||
#[derive(Serialize, Clone)]
|
||||
pub struct TenureEvent {
|
||||
pub year: i32,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
pub struct PropertyData {
|
||||
pub lat: Vec<f32>,
|
||||
pub lon: Vec<f32>,
|
||||
|
|
@ -100,6 +110,9 @@ pub struct PropertyData {
|
|||
/// Per-row historical sale transactions (Land Registry price-paid).
|
||||
/// Keyed by (permuted) row index. Only rows with prices are present.
|
||||
historical_prices: FxHashMap<u32, Vec<HistoricalPrice>>,
|
||||
/// Per-row tenure timeline (owner-occupied <-> rented change points from
|
||||
/// EPC). Keyed by (permuted) row index. Only rows with changes are present.
|
||||
tenure_history: FxHashMap<u32, Vec<TenureEvent>>,
|
||||
property_sub_type: FxHashMap<u32, String>,
|
||||
price_qualifier: FxHashMap<u32, String>,
|
||||
}
|
||||
|
|
@ -153,6 +166,14 @@ impl PropertyData {
|
|||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
/// Get tenure timeline events for a given row (empty slice if none).
|
||||
pub fn tenure_history(&self, row: usize) -> &[TenureEvent] {
|
||||
self.tenure_history
|
||||
.get(&(row as u32))
|
||||
.map(|v| v.as_slice())
|
||||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
/// Get property sub-type for a given row.
|
||||
pub fn property_sub_type(&self, row: usize) -> Option<&str> {
|
||||
self.property_sub_type
|
||||
|
|
@ -231,6 +252,7 @@ impl PropertyData {
|
|||
approx_build_date_bits: Vec::new(),
|
||||
renovation_history: FxHashMap::default(),
|
||||
historical_prices: FxHashMap::default(),
|
||||
tenure_history: FxHashMap::default(),
|
||||
property_sub_type: FxHashMap::default(),
|
||||
price_qualifier: FxHashMap::default(),
|
||||
}
|
||||
|
|
|
|||
127
server-rs/src/demo_zone.rs
Normal file
127
server-rs/src/demo_zone.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//! Per-visitor demo "free zone": the small area an unlicensed visitor may explore.
|
||||
//!
|
||||
//! The visitor's browser chooses the centre — their GPS location (with consent) or
|
||||
//! a Canary Wharf default — and sends it on each request as `demoLat`/`demoLon`
|
||||
//! query params. We build a fixed-size box around that centre for the licensing
|
||||
//! check. Absent/invalid params fall back to Canary Wharf.
|
||||
//!
|
||||
//! The centre is client-supplied and therefore spoofable: this is a soft demo gate
|
||||
//! backed by the rate-limit / filter-cap guards in `ratelimit.rs`, not a hard
|
||||
//! security boundary. (We deliberately removed IP-based geolocation in favour of
|
||||
//! this explicit, consent-based flow.)
|
||||
|
||||
use axum::extract::Request;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use serde::Serialize;
|
||||
use url::form_urlencoded;
|
||||
|
||||
use crate::consts::{
|
||||
FREE_ZONE_FALLBACK_LAT, FREE_ZONE_FALLBACK_LON, FREE_ZONE_HALF_LAT, FREE_ZONE_HALF_LON,
|
||||
};
|
||||
|
||||
/// A demo free-zone bounding box (degrees).
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
pub struct FreeZone {
|
||||
pub south: f64,
|
||||
pub west: f64,
|
||||
pub north: f64,
|
||||
pub east: f64,
|
||||
}
|
||||
|
||||
/// A visitor's resolved demo zone, inserted into request extensions by
|
||||
/// [`demo_zone_middleware`] for every request.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct DemoZone {
|
||||
pub free_zone: FreeZone,
|
||||
}
|
||||
|
||||
/// Build the demo free-zone box centred on `(lat, lon)`, keeping a fixed size and
|
||||
/// clamping to valid lat/lon ranges.
|
||||
pub fn free_zone_around(lat: f64, lon: f64) -> FreeZone {
|
||||
FreeZone {
|
||||
south: (lat - FREE_ZONE_HALF_LAT).max(-90.0),
|
||||
north: (lat + FREE_ZONE_HALF_LAT).min(90.0),
|
||||
west: (lon - FREE_ZONE_HALF_LON).max(-180.0),
|
||||
east: (lon + FREE_ZONE_HALF_LON).min(180.0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the visitor's chosen demo centre from the `demoLat`/`demoLon` query params.
|
||||
fn demo_center_from_query(query: Option<&str>) -> Option<(f64, f64)> {
|
||||
let query = query?;
|
||||
let mut lat: Option<f64> = None;
|
||||
let mut lon: Option<f64> = None;
|
||||
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||
match key.as_ref() {
|
||||
"demoLat" => lat = value.parse().ok(),
|
||||
"demoLon" => lon = value.parse().ok(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let (lat, lon) = (lat?, lon?);
|
||||
if lat.is_finite()
|
||||
&& lon.is_finite()
|
||||
&& (-90.0..=90.0).contains(&lat)
|
||||
&& (-180.0..=180.0).contains(&lon)
|
||||
{
|
||||
Some((lat, lon))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a request's demo zone from its query params, falling back to Canary Wharf.
|
||||
pub fn resolve_demo_zone(query: Option<&str>) -> DemoZone {
|
||||
let (lat, lon) = demo_center_from_query(query)
|
||||
.unwrap_or((FREE_ZONE_FALLBACK_LAT, FREE_ZONE_FALLBACK_LON));
|
||||
DemoZone {
|
||||
free_zone: free_zone_around(lat, lon),
|
||||
}
|
||||
}
|
||||
|
||||
/// Middleware that resolves the visitor's [`DemoZone`] and stashes it in request
|
||||
/// extensions, mirroring how `auth_middleware` provides `OptionalUser`.
|
||||
pub async fn demo_zone_middleware(req: Request, next: Next) -> Response {
|
||||
let zone = resolve_demo_zone(req.uri().query());
|
||||
let (mut parts, body) = req.into_parts();
|
||||
parts.extensions.insert(zone);
|
||||
next.run(Request::from_parts(parts, body)).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_box_is_canary_wharf_sized_and_centred() {
|
||||
let fz = resolve_demo_zone(None).free_zone;
|
||||
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
|
||||
assert!((((fz.west + fz.east) / 2.0) - FREE_ZONE_FALLBACK_LON).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_client_supplied_centre() {
|
||||
let fz = resolve_demo_zone(Some("demoLat=53.4808&demoLon=-2.2426&foo=bar")).free_zone;
|
||||
assert!((((fz.south + fz.north) / 2.0) - 53.4808).abs() < 1e-9);
|
||||
assert!((((fz.west + fz.east) / 2.0) - (-2.2426)).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_out_of_range_or_missing_coords() {
|
||||
// Out of range → fallback.
|
||||
let fz = resolve_demo_zone(Some("demoLat=999&demoLon=0")).free_zone;
|
||||
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
|
||||
// Only one coord → fallback.
|
||||
let fz = resolve_demo_zone(Some("demoLat=53.0")).free_zone;
|
||||
assert!((((fz.south + fz.north) / 2.0) - FREE_ZONE_FALLBACK_LAT).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn box_keeps_a_constant_size_when_relocated() {
|
||||
let a = free_zone_around(51.5, -0.1);
|
||||
let b = free_zone_around(55.0, -3.0);
|
||||
assert!(((a.north - a.south) - (b.north - b.south)).abs() < 1e-9);
|
||||
assert!(((a.east - a.west) - (b.east - b.west)).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
|
|
@ -10,9 +10,10 @@ use url::form_urlencoded;
|
|||
|
||||
use crate::auth::PocketBaseUser;
|
||||
use crate::consts::{
|
||||
FREE_ZONE_BOUNDS, MAX_SHARE_LAT_SPAN, MAX_SHARE_LON_SPAN, MAX_SHARE_ZOOM, MIN_SHARE_ZOOM,
|
||||
SHARE_CACHE_MAX_ENTRIES, SHARE_CACHE_TTL_SECS,
|
||||
MAX_SHARE_LAT_SPAN, MAX_SHARE_LON_SPAN, MAX_SHARE_ZOOM, MIN_SHARE_ZOOM, SHARE_CACHE_MAX_ENTRIES,
|
||||
SHARE_CACHE_TTL_SECS,
|
||||
};
|
||||
use crate::demo_zone::FreeZone;
|
||||
use crate::pocketbase::get_superuser_token;
|
||||
use crate::state::AppState;
|
||||
|
||||
|
|
@ -225,12 +226,14 @@ pub fn is_valid_share_bounds(bounds: ShareBounds) -> bool {
|
|||
|
||||
/// Check whether the user is allowed to query data at the given bounds.
|
||||
/// Licensed users and admins bypass the check entirely.
|
||||
/// Free/anonymous users get 403 unless the bounds fall inside the free zone
|
||||
/// or inside the bbox granted by a valid share code.
|
||||
/// Free/anonymous users get 403 unless the bounds fall inside their free zone
|
||||
/// (a box centred on their IP-approximate location) or inside the bbox granted
|
||||
/// by a valid share code.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn check_license_bounds(
|
||||
user: &Option<PocketBaseUser>,
|
||||
bounds: (f64, f64, f64, f64),
|
||||
free_zone: FreeZone,
|
||||
share_bounds: Option<ShareBounds>,
|
||||
) -> Result<(), axum::response::Response> {
|
||||
if let Some(u) = user {
|
||||
|
|
@ -240,9 +243,12 @@ pub fn check_license_bounds(
|
|||
}
|
||||
|
||||
let (south, west, north, east) = bounds;
|
||||
let (fz_south, fz_west, fz_north, fz_east) = FREE_ZONE_BOUNDS;
|
||||
|
||||
if south >= fz_south && west >= fz_west && north <= fz_north && east <= fz_east {
|
||||
if south >= free_zone.south
|
||||
&& west >= free_zone.west
|
||||
&& north <= free_zone.north
|
||||
&& east <= free_zone.east
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
|
@ -256,10 +262,10 @@ pub fn check_license_bounds(
|
|||
"error": "license_required",
|
||||
"message": "A license is required to view data outside the demo area",
|
||||
"free_zone": {
|
||||
"south": fz_south,
|
||||
"west": fz_west,
|
||||
"north": fz_north,
|
||||
"east": fz_east,
|
||||
"south": free_zone.south,
|
||||
"west": free_zone.west,
|
||||
"north": free_zone.north,
|
||||
"east": free_zone.east,
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -272,9 +278,10 @@ pub fn check_license_point(
|
|||
user: &Option<PocketBaseUser>,
|
||||
lat: f64,
|
||||
lon: f64,
|
||||
free_zone: FreeZone,
|
||||
share_bounds: Option<ShareBounds>,
|
||||
) -> Result<(), axum::response::Response> {
|
||||
check_license_bounds(user, (lat, lon, lat, lon), share_bounds)
|
||||
check_license_bounds(user, (lat, lon, lat, lon), free_zone, share_bounds)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ mod bugsink;
|
|||
mod checkout_sessions;
|
||||
mod consts;
|
||||
mod data;
|
||||
mod demo_zone;
|
||||
mod features;
|
||||
mod language;
|
||||
mod licensing;
|
||||
|
|
@ -15,6 +16,7 @@ mod og_middleware;
|
|||
pub mod parsing;
|
||||
mod pocketbase;
|
||||
mod pocketbase_locks;
|
||||
mod ratelimit;
|
||||
mod routes;
|
||||
mod state;
|
||||
pub mod utils;
|
||||
|
|
@ -720,6 +722,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
stripe_webhook_secret: cli.stripe_webhook_secret,
|
||||
stripe_referral_coupon_id: cli.stripe_referral_coupon_id,
|
||||
bugsink_frontend_config,
|
||||
demo_rate_limiter: Arc::new(ratelimit::DemoRateLimiter::new()),
|
||||
};
|
||||
|
||||
let shared = Arc::new(SharedState::new(app_state));
|
||||
|
|
@ -999,6 +1002,8 @@ async fn main() -> anyhow::Result<()> {
|
|||
api
|
||||
}
|
||||
.layer(middleware::from_fn(metrics::track_metrics))
|
||||
.layer(middleware::from_fn(ratelimit::demo_guard_middleware))
|
||||
.layer(middleware::from_fn(demo_zone::demo_zone_middleware))
|
||||
.layer(middleware::from_fn(auth::auth_middleware))
|
||||
.layer(middleware::from_fn(
|
||||
move |req: axum::extract::Request, next: middleware::Next| {
|
||||
|
|
|
|||
256
server-rs/src/ratelimit.rs
Normal file
256
server-rs/src/ratelimit.rs
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
//! Defense-in-depth guards for unlicensed ("demo") API traffic: a per-key rate
|
||||
//! limit and a server-side filter-count cap. These complement the client-side UX
|
||||
//! limits so a scripted client can't trivially bypass them. Licensed users and
|
||||
//! admins are exempt.
|
||||
//!
|
||||
//! This is a soft anti-abuse measure, not a hard security boundary: the rate-limit
|
||||
//! key falls back to the (spoofable) client IP for anonymous users, and the demo
|
||||
//! gate is best-effort by design (see demo_zone.rs). Logged-in free accounts are keyed
|
||||
//! by their stable account id, which spoofing the IP header does not evade.
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
|
||||
use axum::extract::{ConnectInfo, Request};
|
||||
use axum::http::StatusCode;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use parking_lot::Mutex;
|
||||
use rustc_hash::FxHashMap;
|
||||
use serde_json::json;
|
||||
use url::form_urlencoded;
|
||||
|
||||
use crate::auth::OptionalUser;
|
||||
use crate::consts::{
|
||||
DEMO_MAX_FILTERS, DEMO_RATE_LIMIT_MAX, DEMO_RATE_LIMIT_MAX_KEYS, DEMO_RATE_LIMIT_WINDOW_SECS,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Extract the client IP, preferring reverse-proxy/CDN headers (we sit behind one
|
||||
/// in production) and falling back to the socket peer. Client-supplied and
|
||||
/// spoofable — only used as a coarse rate-limit key for anonymous visitors.
|
||||
fn client_ip(headers: &HeaderMap, peer: Option<IpAddr>) -> Option<IpAddr> {
|
||||
let from_header = |name: &str, take_first: bool| -> Option<IpAddr> {
|
||||
let raw = headers.get(name)?.to_str().ok()?;
|
||||
let candidate = if take_first {
|
||||
raw.split(',').next().unwrap_or(raw)
|
||||
} else {
|
||||
raw
|
||||
};
|
||||
candidate.trim().parse().ok()
|
||||
};
|
||||
|
||||
from_header("cf-connecting-ip", false)
|
||||
.or_else(|| from_header("x-forwarded-for", true))
|
||||
.or_else(|| from_header("x-real-ip", false))
|
||||
.or(peer)
|
||||
}
|
||||
|
||||
/// Sliding-window request counter keyed by account id or client IP.
|
||||
pub struct DemoRateLimiter {
|
||||
windows: Mutex<FxHashMap<String, Vec<Instant>>>,
|
||||
}
|
||||
|
||||
impl DemoRateLimiter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
windows: Mutex::new(FxHashMap::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a hit for `key`; returns `false` when the per-window limit is
|
||||
/// exceeded (caller should reject with 429).
|
||||
pub fn check(&self, key: &str) -> bool {
|
||||
let now = Instant::now();
|
||||
let window = Duration::from_secs(DEMO_RATE_LIMIT_WINDOW_SECS);
|
||||
let mut map = self.windows.lock();
|
||||
|
||||
// Bound memory: when the table grows large, drop keys with no recent hits.
|
||||
// If even that can't get us back under the cap (e.g. a spoofed-IP flood
|
||||
// minting a fresh key per request), clear the table outright. That resets
|
||||
// everyone's window — acceptable under attack — and keeps both the size and
|
||||
// the cost of this scan bounded (it then won't re-run for ~MAX_KEYS inserts,
|
||||
// instead of scanning O(n) on every request once full).
|
||||
if map.len() > DEMO_RATE_LIMIT_MAX_KEYS {
|
||||
map.retain(|_, hits| hits.iter().any(|t| now.duration_since(*t) < window));
|
||||
if map.len() > DEMO_RATE_LIMIT_MAX_KEYS {
|
||||
map.clear();
|
||||
}
|
||||
}
|
||||
|
||||
let hits = map.entry(key.to_string()).or_default();
|
||||
hits.retain(|t| now.duration_since(*t) < window);
|
||||
if hits.len() >= DEMO_RATE_LIMIT_MAX {
|
||||
return false;
|
||||
}
|
||||
hits.push(now);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DemoRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a request is internal/direct rather than from a public visitor via the
|
||||
/// edge proxy. Internal callers (the screenshot/OG service, health checks, local
|
||||
/// dev) reach the server container directly: a private/loopback socket peer and no
|
||||
/// edge-proxy forwarding headers. We exempt them so demo guards never throttle our
|
||||
/// own services. (Behind the public proxy, real visitors carry forwarding headers.)
|
||||
fn is_internal_request(headers: &HeaderMap, peer: Option<IpAddr>) -> bool {
|
||||
let has_forwarding = headers.contains_key("cf-connecting-ip")
|
||||
|| headers.contains_key("x-forwarded-for")
|
||||
|| headers.contains_key("x-real-ip");
|
||||
if has_forwarding {
|
||||
return false;
|
||||
}
|
||||
match peer {
|
||||
Some(IpAddr::V4(v4)) => {
|
||||
v4.is_private() || v4.is_loopback() || v4.is_link_local() || v4.is_unspecified()
|
||||
}
|
||||
Some(IpAddr::V6(v6)) => {
|
||||
v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
|| (v6.segments()[0] & 0xfe00) == 0xfc00 // fc00::/7 unique-local
|
||||
|| (v6.segments()[0] & 0xffc0) == 0xfe80 // fe80::/10 link-local
|
||||
}
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Count non-empty `;;`-separated entries in the `filters` query parameter.
|
||||
fn filter_count(query: &str) -> usize {
|
||||
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||
if key == "filters" {
|
||||
return value.split(";;").filter(|entry| !entry.trim().is_empty()).count();
|
||||
}
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
/// Whether the request carries a non-empty `share` code. Such requests view a shared
|
||||
/// dashboard whose saved filter set may legitimately exceed the demo cap, so they're
|
||||
/// exempt from the filter cap — the actual share-bounds authorization still runs in
|
||||
/// the handler.
|
||||
fn has_share_code(query: &str) -> bool {
|
||||
form_urlencoded::parse(query.as_bytes())
|
||||
.any(|(key, value)| key == "share" && !value.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Middleware applying the demo filter cap and rate limit to unlicensed `/api/`
|
||||
/// traffic. Must run after `auth_middleware` (for `OptionalUser`) and the state
|
||||
/// injection layer (for `AppState`).
|
||||
pub async fn demo_guard_middleware(req: Request, next: Next) -> Response {
|
||||
let path = req.uri().path();
|
||||
// Map/overlay tiles aren't property-data pulls, and the basemap visibly breaks if
|
||||
// they're throttled, so the demo guards only cover the data endpoints.
|
||||
if !path.starts_with("/api/")
|
||||
|| path.starts_with("/api/tiles/")
|
||||
|| path.starts_with("/api/overlays/")
|
||||
{
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
// Licensed users and admins bypass demo guards entirely.
|
||||
let user = req.extensions().get::<OptionalUser>().and_then(|u| u.0.clone());
|
||||
if let Some(u) = &user {
|
||||
if u.is_admin || u.subscription == "licensed" {
|
||||
return next.run(req).await;
|
||||
}
|
||||
}
|
||||
|
||||
let peer = req
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|info| info.0.ip());
|
||||
|
||||
// Exempt our own internal services (screenshots/OG, health checks, dev).
|
||||
if is_internal_request(req.headers(), peer) {
|
||||
return next.run(req).await;
|
||||
}
|
||||
|
||||
// Server-side filter cap (mirrors the client demo limit; blunts oversized
|
||||
// aggregation requests). Share recipients are exempt — their saved view may
|
||||
// legitimately carry >5 filters and the share grant is checked in the handler.
|
||||
if let Some(query) = req.uri().query() {
|
||||
if !has_share_code(query) && filter_count(query) > DEMO_MAX_FILTERS {
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
axum::Json(json!({
|
||||
"error": "filter_limit",
|
||||
"message": format!("The demo is limited to {DEMO_MAX_FILTERS} filters"),
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limit per account (stable, not IP-spoofable) or, for anonymous
|
||||
// visitors, per client IP.
|
||||
let key = match &user {
|
||||
Some(u) => format!("acct:{}", u.id),
|
||||
None => match client_ip(req.headers(), peer) {
|
||||
Some(ip) => format!("ip:{ip}"),
|
||||
None => "anon".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
if let Some(state) = req.extensions().get::<Arc<AppState>>() {
|
||||
if !state.demo_rate_limiter.check(&key) {
|
||||
return (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
axum::Json(json!({
|
||||
"error": "rate_limited",
|
||||
"message": "Too many requests — slow down, or sign in for full access",
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
|
||||
next.run(req).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn counts_filters_in_query() {
|
||||
assert_eq!(filter_count(""), 0);
|
||||
assert_eq!(filter_count("resolution=9"), 0);
|
||||
assert_eq!(filter_count("filters=price%3A1%3A2"), 1);
|
||||
assert_eq!(
|
||||
filter_count("filters=price%3A1%3A2%3B%3Bbeds%3A2%3A4&resolution=9"),
|
||||
2
|
||||
);
|
||||
// Trailing/empty entries are ignored.
|
||||
assert_eq!(filter_count("filters=price%3A1%3A2%3B%3B"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_share_code() {
|
||||
assert!(!has_share_code(""));
|
||||
assert!(!has_share_code("filters=a%3B%3Bb"));
|
||||
assert!(!has_share_code("share=")); // empty value doesn't count
|
||||
assert!(has_share_code("share=abc123"));
|
||||
assert!(has_share_code("bounds=1,2,3,4&share=xyz&filters=a"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limiter_blocks_past_the_window_limit() {
|
||||
let limiter = DemoRateLimiter::new();
|
||||
// The first DEMO_RATE_LIMIT_MAX hits for a key pass; the next is blocked.
|
||||
for _ in 0..DEMO_RATE_LIMIT_MAX {
|
||||
assert!(limiter.check("acct:abc"));
|
||||
}
|
||||
assert!(!limiter.check("acct:abc"));
|
||||
// A different key has its own independent budget.
|
||||
assert!(limiter.check("acct:xyz"));
|
||||
}
|
||||
}
|
||||
|
|
@ -51,6 +51,7 @@ const LISTING_BOUNDS_EPSILON_DEGREES: f64 = 0.00001;
|
|||
pub async fn get_actual_listings(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<ActualListingsParams>,
|
||||
) -> Result<Json<ActualListingsResponse>, Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -70,7 +71,7 @@ pub async fn get_actual_listings(
|
|||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
|
||||
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
|
||||
|
||||
let quant = state.data.quant_ref();
|
||||
let poi_quant = state.data.poi_metrics.quant_ref();
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ use tracing::{info, warn};
|
|||
|
||||
use crate::auth::OptionalUser;
|
||||
use crate::consts::NAN_U16;
|
||||
use crate::data::travel_time::TravelData;
|
||||
use crate::data::{PostcodePoiMetrics, QuantRef};
|
||||
use crate::features;
|
||||
use crate::licensing::{check_license_bounds, resolve_share_code};
|
||||
|
|
@ -369,15 +370,98 @@ fn bounds_for_postcode_indices(
|
|||
(south, west, north, east)
|
||||
}
|
||||
|
||||
/// A travel-time column derived from the active `travel` / `tt` query params.
|
||||
/// Each active travel destination becomes one column showing the per-postcode
|
||||
/// median (or best-case) travel time in minutes.
|
||||
struct TravelColumn {
|
||||
/// Human-readable destination name for the header (e.g. "Bank tube station").
|
||||
header: String,
|
||||
/// Description-row text (e.g. "Travel time by public transport (minutes)").
|
||||
description: String,
|
||||
/// Postcode → travel time data for this destination.
|
||||
data: TravelData,
|
||||
/// Whether to report the best-case (5th percentile) time instead of median.
|
||||
use_best: bool,
|
||||
}
|
||||
|
||||
/// Parse the repeated `tt` frontend-state params into a (mode, slug) → label map.
|
||||
/// Each value looks like `mode:slug:EncodedLabel[:b][:min:max]`, where the label
|
||||
/// was percent-encoded with the frontend's `encodeURIComponent`.
|
||||
fn parse_travel_labels(tt_params: &[String]) -> FxHashMap<(String, String), String> {
|
||||
let mut map = FxHashMap::default();
|
||||
for raw in tt_params {
|
||||
// splitn(4) keeps the label as a single piece; it never contains a raw
|
||||
// ':' because encodeURIComponent escapes it to %3A.
|
||||
let mut parts = raw.splitn(4, ':');
|
||||
let (Some(mode), Some(slug), Some(encoded_label)) =
|
||||
(parts.next(), parts.next(), parts.next())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let label = urlencoding::decode(encoded_label)
|
||||
.map(|cow| cow.into_owned())
|
||||
.unwrap_or_else(|_| encoded_label.to_string());
|
||||
if !label.is_empty() {
|
||||
map.insert((mode.to_string(), slug.to_string()), label);
|
||||
}
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// Turn a destination slug into a display name when no label is supplied.
|
||||
/// "bank-tube-station" → "Bank tube station".
|
||||
fn humanize_slug(slug: &str) -> String {
|
||||
let mut s = slug.replace('-', " ");
|
||||
if let Some(first) = s.get_mut(0..1) {
|
||||
first.make_ascii_uppercase();
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// Friendly name for a travel mode (incl. transit variants) for the desc row.
|
||||
fn pretty_travel_mode(mode: &str) -> &'static str {
|
||||
match mode {
|
||||
"car" => "car",
|
||||
"bicycle" => "bike",
|
||||
"walking" => "walking",
|
||||
m if m.starts_with("transit") => "public transport",
|
||||
_ => "travel",
|
||||
}
|
||||
}
|
||||
|
||||
/// The reported travel time (median, or best-case when `use_best`) for a
|
||||
/// postcode, or None when the destination has no data for it.
|
||||
#[inline]
|
||||
fn travel_minutes_for(data: &TravelData, postcode: &str, use_best: bool) -> Option<i16> {
|
||||
data.get(postcode).map(|row| {
|
||||
if use_best {
|
||||
row.best_minutes.unwrap_or(row.minutes)
|
||||
} else {
|
||||
row.minutes
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_export(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
headers: HeaderMap,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
uri: Uri,
|
||||
Query(params): Query<ExportParams>,
|
||||
) -> Result<impl IntoResponse, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
||||
// Exporting requires an account — no anonymous/demo exports.
|
||||
if user.0.is_none() {
|
||||
return Err((
|
||||
StatusCode::UNAUTHORIZED,
|
||||
[(header::CONTENT_TYPE, "application/json")],
|
||||
r#"{"error":"account_required","message":"Sign in to export data"}"#,
|
||||
)
|
||||
.into_response());
|
||||
}
|
||||
|
||||
// Two modes: bounds-based (default) and explicit postcode list.
|
||||
let postcode_list = match params.postcodes.as_deref() {
|
||||
Some(raw) if !raw.trim().is_empty() => {
|
||||
|
|
@ -418,7 +502,7 @@ pub async fn get_export(
|
|||
}
|
||||
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
|
||||
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
|
||||
|
||||
let quant = state.data.quant_ref();
|
||||
let poi_quant = state.data.poi_metrics.quant_ref();
|
||||
|
|
@ -450,20 +534,17 @@ pub async fn get_export(
|
|||
} else {
|
||||
params.filters
|
||||
};
|
||||
let travel_entries = if is_postcode_mode {
|
||||
Vec::new()
|
||||
} else {
|
||||
parse_optional_travel(params.travel.as_deref())
|
||||
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
|
||||
};
|
||||
let has_travel_filters = travel_entries
|
||||
.iter()
|
||||
.any(|entry| entry.filter_min.is_some() && entry.filter_max.is_some());
|
||||
let travel_state_params = if is_postcode_mode {
|
||||
Vec::new()
|
||||
} else {
|
||||
collect_travel_state_params(uri.query())
|
||||
};
|
||||
// Travel times are per-postcode global data, so they appear as columns in
|
||||
// both bounds and list mode. The time-range *filter* is only applied on the
|
||||
// bounds path (`has_travel_filters` below); list mode never filters rows, so
|
||||
// every requested postcode still shows up with its travel time.
|
||||
let travel_entries = parse_optional_travel(params.travel.as_deref())
|
||||
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
|
||||
let has_travel_filters = !is_postcode_mode
|
||||
&& travel_entries
|
||||
.iter()
|
||||
.any(|entry| entry.filter_min.is_some() && entry.filter_max.is_some());
|
||||
let travel_state_params = collect_travel_state_params(uri.query());
|
||||
let overlay_state_params = if is_postcode_mode {
|
||||
Vec::new()
|
||||
} else {
|
||||
|
|
@ -553,6 +634,33 @@ pub async fn get_export(
|
|||
let postcode_data = &state.postcode_data;
|
||||
let poi_metrics = &state.data.poi_metrics;
|
||||
let travel_data = load_travel_data(&state.travel_time_store, &travel_entries)?;
|
||||
|
||||
// One column per active travel destination. `travel_data` is built by
|
||||
// mapping over `travel_entries` in order, so the two are index-aligned.
|
||||
// Display labels come from the `tt` frontend-state params.
|
||||
let travel_labels = parse_travel_labels(&travel_state_params);
|
||||
let travel_columns: Vec<TravelColumn> = travel_entries
|
||||
.iter()
|
||||
.zip(travel_data.iter())
|
||||
.map(|(entry, data)| {
|
||||
let header = travel_labels
|
||||
.get(&(entry.mode.clone(), entry.slug.clone()))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| humanize_slug(&entry.slug));
|
||||
let description = format!(
|
||||
"Travel time by {}{} (minutes)",
|
||||
pretty_travel_mode(&entry.mode),
|
||||
if entry.use_best { ", best case" } else { "" }
|
||||
);
|
||||
TravelColumn {
|
||||
header,
|
||||
description,
|
||||
data: Arc::clone(data),
|
||||
use_best: entry.use_best,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let poi_offset = num_features;
|
||||
let total_export_features = num_features + poi_metrics.num_features();
|
||||
let (pc_interner, pc_keys) = state.data.postcode_parts();
|
||||
|
|
@ -701,6 +809,15 @@ pub async fn get_export(
|
|||
ordered
|
||||
};
|
||||
|
||||
// The "All Data" sheet lists property/area features only. POI amenity
|
||||
// counts & distances (virtual indices >= poi_offset) are reserved for the
|
||||
// "Selected" sheet, where they show up when the user filters/selects them.
|
||||
let all_data_feature_indices: Vec<usize> = all_feature_indices
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&idx| idx < poi_offset)
|
||||
.collect();
|
||||
|
||||
// Filter-only feature indices for the Selected sheet
|
||||
let filter_feature_indices: Vec<usize> = filter_feature_names
|
||||
.iter()
|
||||
|
|
@ -839,6 +956,12 @@ pub async fn get_export(
|
|||
let group_label_fmt = Format::new().set_bold().set_font_color("#1F4E79");
|
||||
let group_count_fmt = Format::new().set_bold();
|
||||
|
||||
// Travel-time cells render the integer minute count with a " min" suffix.
|
||||
let travel_num_fmt = Format::new().set_num_format("0\" min\"");
|
||||
|
||||
// Postcode-centroid coordinates (WGS84), ~1 m precision at 5 decimals.
|
||||
let coord_num_fmt = Format::new().set_num_format("0.00000");
|
||||
|
||||
// Dashboard URL
|
||||
let dashboard_url = format!(
|
||||
"{}/dashboard?{}",
|
||||
|
|
@ -847,14 +970,16 @@ pub async fn get_export(
|
|||
);
|
||||
|
||||
// Two sheets in both modes: "Selected" (just the features behind the
|
||||
// active filters) and "All Data" (every feature). The Selected sheet
|
||||
// carries the dashboard link + screenshot only in bounds mode, where the
|
||||
// export is tied to a map view; in list mode the user picked the
|
||||
// postcodes explicitly, so there's no map view to link or screenshot.
|
||||
// active filters, including any POI amenity metrics) and "All Data"
|
||||
// (every property/area feature, but no POI amenity counts/distances —
|
||||
// those live only on the Selected sheet). The Selected sheet carries the
|
||||
// dashboard link + screenshot only in bounds mode, where the export is
|
||||
// tied to a map view; in list mode the user picked the postcodes
|
||||
// explicitly, so there's no map view to link or screenshot.
|
||||
let is_list_mode = postcode_list_entries.is_some();
|
||||
let sheet_configs: Vec<(&str, &[usize], bool)> = vec![
|
||||
("Selected", &filter_feature_indices, !is_list_mode),
|
||||
("All Data", &all_feature_indices, false),
|
||||
("All Data", &all_data_feature_indices, false),
|
||||
];
|
||||
|
||||
for (sheet_name, feat_indices, include_header) in &sheet_configs {
|
||||
|
|
@ -904,6 +1029,10 @@ pub async fn get_export(
|
|||
|
||||
// Header row
|
||||
let header_row = current_row;
|
||||
// Travel-time columns sit to the right of the feature columns, with
|
||||
// the Latitude/Longitude pair last.
|
||||
let travel_col_base = (feat_indices.len() + 2) as u16;
|
||||
let coord_col_base = travel_col_base + travel_columns.len() as u16;
|
||||
sheet
|
||||
.write_string_with_format(header_row, 0, "Postcode", &header_fmt)
|
||||
.map_err(|e| format!("Failed to write header: {e}"))?;
|
||||
|
|
@ -923,6 +1052,24 @@ pub async fn get_export(
|
|||
.map_err(|e| format!("Failed to write header: {e}"))?;
|
||||
}
|
||||
|
||||
for (i, tc) in travel_columns.iter().enumerate() {
|
||||
sheet
|
||||
.write_string_with_format(
|
||||
header_row,
|
||||
travel_col_base + i as u16,
|
||||
&tc.header,
|
||||
&header_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write travel header: {e}"))?;
|
||||
}
|
||||
|
||||
sheet
|
||||
.write_string_with_format(header_row, coord_col_base, "Latitude", &header_fmt)
|
||||
.map_err(|e| format!("Failed to write header: {e}"))?;
|
||||
sheet
|
||||
.write_string_with_format(header_row, coord_col_base + 1, "Longitude", &header_fmt)
|
||||
.map_err(|e| format!("Failed to write header: {e}"))?;
|
||||
|
||||
// Description row
|
||||
let desc_row = header_row + 1;
|
||||
sheet
|
||||
|
|
@ -943,6 +1090,29 @@ pub async fn get_export(
|
|||
.map_err(|e| format!("Failed to write desc: {e}"))?;
|
||||
}
|
||||
|
||||
for (i, tc) in travel_columns.iter().enumerate() {
|
||||
sheet
|
||||
.write_string_with_format(
|
||||
desc_row,
|
||||
travel_col_base + i as u16,
|
||||
&tc.description,
|
||||
&desc_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write travel desc: {e}"))?;
|
||||
}
|
||||
|
||||
sheet
|
||||
.write_string_with_format(desc_row, coord_col_base, "Postcode centroid", &desc_fmt)
|
||||
.map_err(|e| format!("Failed to write desc: {e}"))?;
|
||||
sheet
|
||||
.write_string_with_format(
|
||||
desc_row,
|
||||
coord_col_base + 1,
|
||||
"Postcode centroid",
|
||||
&desc_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write desc: {e}"))?;
|
||||
|
||||
// Put the collapse/expand controls above each group so the bold
|
||||
// outcode summary row acts as the header for its postcodes.
|
||||
sheet.group_symbols_above(true);
|
||||
|
|
@ -976,6 +1146,63 @@ pub async fn get_export(
|
|||
&integer_feature_indices,
|
||||
&feat_num_fmts,
|
||||
)?;
|
||||
// Travel time rolled up across the group's postcodes, weighted by
|
||||
// property count to match the property-weighted feature means.
|
||||
for (i, tc) in travel_columns.iter().enumerate() {
|
||||
let mut weighted_sum = 0.0_f64;
|
||||
let mut weight = 0u32;
|
||||
for &member in &group.members {
|
||||
let (member_pc_idx, member_agg) = &postcode_aggs[member];
|
||||
let pc = &postcode_data.postcodes[*member_pc_idx];
|
||||
if let Some(minutes) = travel_minutes_for(&tc.data, pc, tc.use_best) {
|
||||
weighted_sum += minutes as f64 * member_agg.count as f64;
|
||||
weight += member_agg.count;
|
||||
}
|
||||
}
|
||||
if weight > 0 {
|
||||
let mean = (weighted_sum / weight as f64).round();
|
||||
sheet
|
||||
.write_number_with_format(
|
||||
summary_row,
|
||||
travel_col_base + i as u16,
|
||||
mean,
|
||||
&travel_num_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write travel summary: {e}"))?;
|
||||
}
|
||||
}
|
||||
// Property-weighted centroid of the group's postcodes.
|
||||
{
|
||||
let mut lat_sum = 0.0_f64;
|
||||
let mut lon_sum = 0.0_f64;
|
||||
let mut weight = 0u32;
|
||||
for &member in &group.members {
|
||||
let (member_pc_idx, member_agg) = &postcode_aggs[member];
|
||||
let (lat, lon) = postcode_data.centroids[*member_pc_idx];
|
||||
lat_sum += lat as f64 * member_agg.count as f64;
|
||||
lon_sum += lon as f64 * member_agg.count as f64;
|
||||
weight += member_agg.count;
|
||||
}
|
||||
if weight > 0 {
|
||||
let w = weight as f64;
|
||||
sheet
|
||||
.write_number_with_format(
|
||||
summary_row,
|
||||
coord_col_base,
|
||||
lat_sum / w,
|
||||
&coord_num_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write latitude: {e}"))?;
|
||||
sheet
|
||||
.write_number_with_format(
|
||||
summary_row,
|
||||
coord_col_base + 1,
|
||||
lon_sum / w,
|
||||
&coord_num_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write longitude: {e}"))?;
|
||||
}
|
||||
}
|
||||
row += 1;
|
||||
|
||||
// Individual postcode rows for this outcode.
|
||||
|
|
@ -999,6 +1226,34 @@ pub async fn get_export(
|
|||
&integer_feature_indices,
|
||||
&feat_num_fmts,
|
||||
)?;
|
||||
for (i, tc) in travel_columns.iter().enumerate() {
|
||||
if let Some(minutes) = travel_minutes_for(
|
||||
&tc.data,
|
||||
&postcode_data.postcodes[*pc_idx],
|
||||
tc.use_best,
|
||||
) {
|
||||
sheet
|
||||
.write_number_with_format(
|
||||
row,
|
||||
travel_col_base + i as u16,
|
||||
minutes as f64,
|
||||
&travel_num_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write travel time: {e}"))?;
|
||||
}
|
||||
}
|
||||
let (lat, lon) = postcode_data.centroids[*pc_idx];
|
||||
sheet
|
||||
.write_number_with_format(row, coord_col_base, lat as f64, &coord_num_fmt)
|
||||
.map_err(|e| format!("Failed to write latitude: {e}"))?;
|
||||
sheet
|
||||
.write_number_with_format(
|
||||
row,
|
||||
coord_col_base + 1,
|
||||
lon as f64,
|
||||
&coord_num_fmt,
|
||||
)
|
||||
.map_err(|e| format!("Failed to write longitude: {e}"))?;
|
||||
row += 1;
|
||||
}
|
||||
|
||||
|
|
@ -1012,7 +1267,8 @@ pub async fn get_export(
|
|||
// Sample note
|
||||
if was_sampled {
|
||||
let note_row = row + 1;
|
||||
let total_cols = (feat_indices.len() + 2) as u16;
|
||||
// +2 for the Latitude/Longitude columns.
|
||||
let total_cols = (feat_indices.len() + 2 + travel_columns.len() + 2) as u16;
|
||||
sheet
|
||||
.merge_range(
|
||||
note_row,
|
||||
|
|
@ -1043,6 +1299,18 @@ pub async fn get_export(
|
|||
.set_column_width(col, width)
|
||||
.map_err(|e| format!("Failed to set column width: {e}"))?;
|
||||
}
|
||||
for (i, tc) in travel_columns.iter().enumerate() {
|
||||
let width = (tc.header.chars().count() as f64 * 1.1).clamp(10.0, 30.0);
|
||||
sheet
|
||||
.set_column_width(travel_col_base + i as u16, width)
|
||||
.map_err(|e| format!("Failed to set travel column width: {e}"))?;
|
||||
}
|
||||
sheet
|
||||
.set_column_width(coord_col_base, 11)
|
||||
.map_err(|e| format!("Failed to set coordinate column width: {e}"))?;
|
||||
sheet
|
||||
.set_column_width(coord_col_base + 1, 11)
|
||||
.map_err(|e| format!("Failed to set coordinate column width: {e}"))?;
|
||||
}
|
||||
|
||||
let buf = workbook
|
||||
|
|
@ -1128,6 +1396,50 @@ mod tests {
|
|||
assert_eq!(outcode_of("E14"), "E14");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_travel_labels_decodes_label_keyed_by_mode_and_slug() {
|
||||
// As produced by collect_travel_state_params (outer URL layer already
|
||||
// decoded); the label is still encodeURIComponent-encoded.
|
||||
let params = vec![
|
||||
"transit:bank-tube-station:Bank%20tube%20station:0:52".to_string(),
|
||||
"car:heathrow-airport:Heathrow%20Airport:b:0:90".to_string(),
|
||||
];
|
||||
let labels = parse_travel_labels(¶ms);
|
||||
assert_eq!(
|
||||
labels.get(&("transit".to_string(), "bank-tube-station".to_string())),
|
||||
Some(&"Bank tube station".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
labels.get(&("car".to_string(), "heathrow-airport".to_string())),
|
||||
Some(&"Heathrow Airport".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_travel_labels_skips_entries_without_a_label() {
|
||||
// Empty label (encodeURIComponent("")) and a too-short value are ignored.
|
||||
let params = vec![
|
||||
"transit:bank-tube-station::b:0:52".to_string(),
|
||||
"car:somewhere".to_string(),
|
||||
];
|
||||
assert!(parse_travel_labels(¶ms).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn humanize_slug_falls_back_to_a_readable_name() {
|
||||
assert_eq!(humanize_slug("bank-tube-station"), "Bank tube station");
|
||||
assert_eq!(humanize_slug("london"), "London");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pretty_travel_mode_maps_transit_variants() {
|
||||
assert_eq!(pretty_travel_mode("transit"), "public transport");
|
||||
assert_eq!(pretty_travel_mode("transit-no-buses"), "public transport");
|
||||
assert_eq!(pretty_travel_mode("car"), "car");
|
||||
assert_eq!(pretty_travel_mode("bicycle"), "bike");
|
||||
assert_eq!(pretty_travel_mode("walking"), "walking");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_query_deserializes_when_tt_is_a_single_string() {
|
||||
let uri: Uri = "/api/export?bounds=1,2,3,4&tt=transit%3Abank%3ABank%2520station%3A0%3A52"
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ pub struct FilterCountsResponse {
|
|||
pub async fn get_filter_counts(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<FilterCountsParams>,
|
||||
) -> Result<Json<FilterCountsResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -40,7 +41,7 @@ pub async fn get_filter_counts(
|
|||
let (south, west, north, east) =
|
||||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
|
||||
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
|
||||
|
||||
let quant = state.data.quant_ref();
|
||||
let poi_quant = state.data.poi_metrics.quant_ref();
|
||||
|
|
|
|||
|
|
@ -436,6 +436,7 @@ pub(super) fn top_filter_exclusions(
|
|||
pub async fn get_hexagon_stats(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<HexagonStatsParams>,
|
||||
) -> Result<Json<HexagonStatsResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -455,7 +456,7 @@ pub async fn get_hexagon_stats(
|
|||
// License check using H3 cell bounds
|
||||
let h3_bounds = h3_cell_bounds(cell, 0.0);
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, h3_bounds, share_bounds)?;
|
||||
check_license_bounds(&user.0, h3_bounds, geo.free_zone, share_bounds)?;
|
||||
|
||||
let h3_str = params.h3;
|
||||
let quant = state.data.quant_ref();
|
||||
|
|
|
|||
|
|
@ -252,6 +252,7 @@ fn build_feature_maps(
|
|||
pub async fn get_hexagons(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<HexagonParams>,
|
||||
) -> Result<Json<HexagonsResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -262,7 +263,7 @@ pub async fn get_hexagons(
|
|||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
|
||||
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
|
||||
|
||||
let quant = state.data.quant_ref();
|
||||
let poi_quant = state.data.poi_metrics.quant_ref();
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ fn destination_coordinates(place_data: &PlaceData, slug: &str) -> Option<(f32, f
|
|||
pub async fn get_journey(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
query: axum::extract::Query<JourneyQuery>,
|
||||
) -> Result<Json<JourneyResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -72,7 +73,7 @@ pub async fn get_journey(
|
|||
let (lat, lon) = state.postcode_data.centroids[pc_idx];
|
||||
|
||||
let share_bounds = resolve_share_code(&state, query.share.as_deref()).await;
|
||||
check_license_point(&user.0, lat as f64, lon as f64, share_bounds)?;
|
||||
check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
|
||||
|
||||
if !store.has_destination(&query.mode, &query.slug) {
|
||||
return Err((
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ pub struct PostcodePropertiesParams {
|
|||
pub async fn get_postcode_properties(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<PostcodePropertiesParams>,
|
||||
) -> Result<Json<PropertyListResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -64,6 +65,7 @@ pub async fn get_postcode_properties(
|
|||
&user.0,
|
||||
centroid_lat as f64,
|
||||
centroid_lon as f64,
|
||||
geo.free_zone,
|
||||
share_bounds,
|
||||
)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ use crate::state::SharedState;
|
|||
use crate::utils::normalize_postcode;
|
||||
|
||||
use super::hexagon_stats::{
|
||||
parse_area_stats_field_set, top_filter_exclusions, HexagonStatsResponse,
|
||||
parse_area_stats_field_set, top_filter_exclusions, EnumFeatureStats, HexagonStatsResponse,
|
||||
};
|
||||
use super::stats;
|
||||
use super::travel_time::{load_travel_data, parse_optional_travel, row_passes_travel_filters};
|
||||
|
|
@ -38,6 +38,7 @@ pub struct PostcodeStatsParams {
|
|||
pub async fn get_postcode_stats(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<PostcodeStatsParams>,
|
||||
) -> Result<Json<HexagonStatsResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -63,6 +64,7 @@ pub async fn get_postcode_stats(
|
|||
&user.0,
|
||||
centroid_lat as f64,
|
||||
centroid_lon as f64,
|
||||
geo.free_zone,
|
||||
share_bounds,
|
||||
)?;
|
||||
|
||||
|
|
@ -157,7 +159,7 @@ pub async fn get_postcode_stats(
|
|||
&field_set,
|
||||
);
|
||||
|
||||
let (mut numeric_features, enum_features_out) = stats::compute_feature_stats(
|
||||
let (mut numeric_features, mut enum_features_out) = stats::compute_feature_stats(
|
||||
&matching_rows,
|
||||
&state.data,
|
||||
&state.data.feature_names,
|
||||
|
|
@ -173,6 +175,31 @@ pub async fn get_postcode_stats(
|
|||
&field_set,
|
||||
));
|
||||
|
||||
// The council-house count is an attribute of the postcode itself, not of
|
||||
// the currently filter-matching subset: recompute it over every property
|
||||
// in the postcode so toggling filters never changes it. (When no filters
|
||||
// are applied, `area_rows == matching_rows` and this is a no-op.)
|
||||
if let Some(&council_idx) = state
|
||||
.feature_name_to_index
|
||||
.get(crate::consts::FORMER_COUNCIL_HOUSE_FEATURE)
|
||||
{
|
||||
enum_features_out.retain(|f| f.name != crate::consts::FORMER_COUNCIL_HOUSE_FEATURE);
|
||||
let included = !fields_specified
|
||||
|| field_set.contains(crate::consts::FORMER_COUNCIL_HOUSE_FEATURE);
|
||||
if included {
|
||||
if let Some(counts) =
|
||||
stats::compute_enum_feature_counts(&area_rows, &state.data, council_idx)
|
||||
{
|
||||
if !counts.is_empty() {
|
||||
enum_features_out.push(EnumFeatureStats {
|
||||
name: crate::consts::FORMER_COUNCIL_HOUSE_FEATURE.to_string(),
|
||||
counts,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(
|
||||
postcode = %postcode_str,
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ pub struct PostcodeParams {
|
|||
pub async fn get_postcodes(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<PostcodeParams>,
|
||||
) -> Result<Json<PostcodesResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -61,7 +62,7 @@ pub async fn get_postcodes(
|
|||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, (south, west, north, east), share_bounds)?;
|
||||
check_license_bounds(&user.0, (south, west, north, east), geo.free_zone, share_bounds)?;
|
||||
|
||||
let quant = state.data.quant_ref();
|
||||
let poi_quant = state.data.poi_metrics.quant_ref();
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use tracing::{info, warn};
|
|||
|
||||
use crate::auth::OptionalUser;
|
||||
use crate::consts::PROPERTIES_LIMIT;
|
||||
use crate::data::{HistoricalPrice, RenovationEvent};
|
||||
use crate::data::{HistoricalPrice, RenovationEvent, TenureEvent};
|
||||
use crate::licensing::{check_license_bounds, resolve_share_code};
|
||||
use crate::parsing::{
|
||||
cell_for_row_cached, h3_cell_bounds, needs_parent, parse_field_indices_with_poi,
|
||||
|
|
@ -67,6 +67,9 @@ pub struct Property {
|
|||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub historical_prices: Vec<HistoricalPrice>,
|
||||
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub tenure_history: Vec<TenureEvent>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub features: FxHashMap<String, f32>,
|
||||
}
|
||||
|
|
@ -240,6 +243,7 @@ pub fn build_property(
|
|||
lon: state.data.lon[row],
|
||||
renovation_history: state.data.renovation_history(row).to_vec(),
|
||||
historical_prices: state.data.historical_prices(row).to_vec(),
|
||||
tenure_history: state.data.tenure_history(row).to_vec(),
|
||||
property_sub_type: state.data.property_sub_type(row).map(String::from),
|
||||
price_qualifier: state.data.price_qualifier(row).map(String::from),
|
||||
former_council_house: lookup_enum_value(
|
||||
|
|
@ -270,6 +274,7 @@ pub fn build_property(
|
|||
pub async fn get_hexagon_properties(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<HexagonPropertiesParams>,
|
||||
) -> Result<Json<PropertyListResponse>, axum::response::Response> {
|
||||
let state = shared.load_state();
|
||||
|
|
@ -289,7 +294,7 @@ pub async fn get_hexagon_properties(
|
|||
// License check using H3 cell bounds
|
||||
let h3_bounds = h3_cell_bounds(cell, 0.0);
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(&user.0, h3_bounds, share_bounds)?;
|
||||
check_license_bounds(&user.0, h3_bounds, geo.free_zone, share_bounds)?;
|
||||
|
||||
let h3_str = params.h3;
|
||||
let quant = state.data.quant_ref();
|
||||
|
|
|
|||
|
|
@ -255,6 +255,36 @@ pub fn compute_feature_stats(
|
|||
(numeric_features, enum_features_out)
|
||||
}
|
||||
|
||||
/// Count occurrences of each variant of a single enum feature across `rows`.
|
||||
///
|
||||
/// Unlike [`compute_feature_stats`], which is driven by the filter-matching
|
||||
/// subset, this lets a caller compute a count that should reflect a whole area
|
||||
/// regardless of the active filters (e.g. the council-house count, which is an
|
||||
/// attribute of the postcode itself, not of the currently filtered properties).
|
||||
/// Returns `None` if `feature_idx` is not an enum feature.
|
||||
pub fn compute_enum_feature_counts(
|
||||
rows: &[usize],
|
||||
data: &PropertyData,
|
||||
feature_idx: usize,
|
||||
) -> Option<HashMap<String, u64>> {
|
||||
let variants = data.enum_values.get(&feature_idx)?;
|
||||
let mut value_counts = vec![0u64; variants.len()];
|
||||
for &row in rows {
|
||||
let value = data.get_feature(row, feature_idx);
|
||||
if value.is_finite() && value >= 0.0 && (value as usize) < value_counts.len() {
|
||||
value_counts[value as usize] += 1;
|
||||
}
|
||||
}
|
||||
Some(
|
||||
value_counts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, &count)| count > 0)
|
||||
.map(|(idx, &count)| (variants[idx].clone(), count))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Compute property-weighted per-year crime means across the selection.
|
||||
///
|
||||
/// Each matching property contributes its postcode's per-year counts (incidents
|
||||
|
|
@ -447,3 +477,42 @@ pub fn compute_poi_feature_stats(
|
|||
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::consts::NAN_U16;
|
||||
|
||||
fn enum_data(values: &[u16]) -> PropertyData {
|
||||
let mut data = PropertyData::empty_for_tests();
|
||||
data.num_features = 1;
|
||||
data.num_numeric = 0; // single enum feature at index 0
|
||||
data.feature_data = values.to_vec();
|
||||
data.enum_values
|
||||
.insert(0, vec!["Yes".to_string(), "No".to_string()]);
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enum_counts_tally_only_given_rows() {
|
||||
// Rows: Yes, No, Yes, <missing>
|
||||
let data = enum_data(&[0, 1, 0, NAN_U16]);
|
||||
|
||||
let all = compute_enum_feature_counts(&[0, 1, 2, 3], &data, 0).unwrap();
|
||||
assert_eq!(all.get("Yes"), Some(&2));
|
||||
assert_eq!(all.get("No"), Some(&1));
|
||||
|
||||
// A filter-matching subset would yield a different tally — confirming
|
||||
// the count is driven purely by the rows passed in (so callers can pass
|
||||
// the full area to make it filter-independent).
|
||||
let subset = compute_enum_feature_counts(&[0, 3], &data, 0).unwrap();
|
||||
assert_eq!(subset.get("Yes"), Some(&1));
|
||||
assert_eq!(subset.get("No"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enum_counts_none_for_non_enum_feature() {
|
||||
let data = enum_data(&[0, 1]);
|
||||
assert!(compute_enum_feature_counts(&[0, 1], &data, 1).is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ use crate::utils::GridIndex;
|
|||
pub struct AppState {
|
||||
pub data: PropertyData,
|
||||
pub grid: GridIndex,
|
||||
/// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (12).
|
||||
/// h3_cells[row_idx] = precomputed H3 cell ID at max resolution (9).
|
||||
/// Parent cells for lower resolutions derived via CellIndex::parent().
|
||||
pub h3_cells: Vec<u64>,
|
||||
/// O(1) lookup: feature name → index in feature_names/feature_data
|
||||
|
|
@ -85,6 +85,8 @@ pub struct AppState {
|
|||
pub stripe_referral_coupon_id: String,
|
||||
/// Bugsink/Sentry-compatible browser error reporting config injected into served HTML.
|
||||
pub bugsink_frontend_config: Option<BugsinkFrontendConfig>,
|
||||
/// Per-key rate limiter for unlicensed ("demo") API traffic.
|
||||
pub demo_rate_limiter: Arc<crate::ratelimit::DemoRateLimiter>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -182,6 +184,7 @@ impl AppState {
|
|||
stripe_webhook_secret: "whsec_test_secret".to_string(),
|
||||
stripe_referral_coupon_id: "couponTest30".to_string(),
|
||||
bugsink_frontend_config: None,
|
||||
demo_rate_limiter: Arc::new(crate::ratelimit::DemoRateLimiter::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue