andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

vibes

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-15 17:38:26 +00:00
parent 80c093b7ba
commit f72c43a9fa
101 changed files with 2168 additions and 1177 deletions
Download patch file Download diff file Expand all files Collapse all files

28
server-rs/src/aggregation.rs
View file

@ -1,3 +1,6 @@
use crate::consts::NAN_U16;
use crate::data::QuantRef;
/// Per-cell accumulator for aggregating features (min/max/sum/count).
/// Uses Box<[T]> instead of Vec<T> to avoid storing capacity (saves 8 bytes per field per cell).
/// Shared by hexagon and postcode aggregation routes.
@ -20,16 +23,23 @@ impl Aggregator {
}
}
/// Add a row using row-major feature_data layout.
/// Add a row using row-major feature_data layout (quantized u16).
/// feature_data[row * num_features + feat_idx] — all features for one row
/// are contiguous, so this reads a single cache line per ~8 features.
/// are contiguous, so this reads a single cache line per ~16 features.
#[inline]
pub fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
pub fn add_row(
&mut self,
feature_data: &[u16],
row: usize,
num_features: usize,
quant: &QuantRef,
) {
self.count += 1;
let base = row * num_features;
let row_slice = &feature_data[base..base + num_features];
for (feat_index, &value) in row_slice.iter().enumerate() {
if value.is_finite() {
for (feat_index, &raw) in row_slice.iter().enumerate() {
if raw != NAN_U16 {
let value = quant.decode(feat_index, raw);
if value < self.mins[feat_index] {
self.mins[feat_index] = value;
}
@ -46,16 +56,18 @@ impl Aggregator {
#[inline]
pub fn add_row_selective(
&mut self,
feature_data: &[f32],
feature_data: &[u16],
row: usize,
num_features: usize,
indices: &[usize],
quant: &QuantRef,
) {
self.count += 1;
let base = row * num_features;
for &feat_index in indices {
let value = feature_data[base + feat_index];
if value.is_finite() {
let raw = feature_data[base + feat_index];
if raw != NAN_U16 {
let value = quant.decode(feat_index, raw);
if value < self.mins[feat_index] {
self.mins[feat_index] = value;
}

3
server-rs/src/consts.rs
View file

@ -1,3 +1,6 @@
pub const NAN_U16: u16 = u16::MAX;
pub const QUANT_SCALE: f32 = 65534.0;
pub const HISTOGRAM_BINS: usize = 100;
pub const H3_PRECOMPUTE_MAX: u8 = 12;

4
server-rs/src/data.rs
View file

@ -7,5 +7,7 @@ pub mod travel_time;
pub use places::PlaceData;
pub use poi::{POICategoryGroup, POIData};
pub use postcodes::PostcodeData;
pub use property::{precompute_h3, FeatureStats, Histogram, PropertyData, RenovationEvent};
pub use property::{
precompute_h3, FeatureStats, Histogram, PropertyData, QuantRef, RenovationEvent,
};
pub use travel_time::{slugify, TravelTimeStore};

35
server-rs/src/data/poi.rs
View file

@ -18,7 +18,12 @@ pub struct POICategoryGroup {
}
pub struct POIData {
pub id: Vec<String>,
/// Contiguous buffer holding all POI ID strings end-to-end.
id_buffer: String,
/// Byte offset into `id_buffer` where each row's ID starts.
id_offsets: Vec<u32>,
/// Length in bytes of each row's ID.
id_lengths: Vec<u8>,
pub group: InternedColumn,
pub category: InternedColumn,
pub name: Vec<String>,
@ -31,6 +36,15 @@ pub struct POIData {
pub priority: Vec<u32>,
}
impl POIData {
/// Get the ID string for a given row.
pub fn id(&self, row: usize) -> &str {
let offset = self.id_offsets[row] as usize;
let length = self.id_lengths[row] as usize;
&self.id_buffer[offset..offset + length]
}
}
fn extract_str_col(df: &DataFrame, name: &str) -> anyhow::Result<Vec<String>> {
let column = df
.column(name)
@ -72,7 +86,7 @@ impl POIData {
let row_count = df.height();
info!("Loaded {} POIs", row_count);
let id: Vec<String> = extract_str_col(&df, "id")?;
let id_raw: Vec<String> = extract_str_col(&df, "id")?;
let name = extract_str_col(&df, "name")?;
let category_raw = extract_str_col(&df, "category")?;
let group_raw = extract_str_col(&df, "group")?;
@ -80,6 +94,19 @@ impl POIData {
let lng = extract_f32_col(&df, "lng", 0.0)?;
let emoji_raw = extract_str_col(&df, "emoji")?;
// Pack POI IDs into a contiguous buffer
let total_id_bytes: usize = id_raw.iter().map(|s| s.len()).sum();
let mut id_buffer = String::with_capacity(total_id_bytes);
let mut id_offsets = Vec::with_capacity(row_count);
let mut id_lengths = Vec::with_capacity(row_count);
for s in &id_raw {
let offset = id_buffer.len() as u32;
let length = s.len().min(u8::MAX as usize) as u8;
id_offsets.push(offset);
id_lengths.push(length);
id_buffer.push_str(&s[..length as usize]);
}
let category = InternedColumn::build(&category_raw);
let group = InternedColumn::build(&group_raw);
let emoji = InternedColumn::build(&emoji_raw);
@ -99,7 +126,9 @@ impl POIData {
info!("POI data loading complete.");
Ok(POIData {
id,
id_buffer,
id_offsets,
id_lengths,
name,
category,
group,

244
server-rs/src/data/property.rs
View file

@ -7,7 +7,7 @@ use std::path::Path;
use rustc_hash::FxHashMap;
use crate::consts::{H3_PRECOMPUTE_MAX, HISTOGRAM_BINS};
use crate::consts::{H3_PRECOMPUTE_MAX, HISTOGRAM_BINS, NAN_U16, QUANT_SCALE};
use crate::features::{self, Bounds};
fn is_numeric_dtype(dtype: &DataType) -> bool {
@ -47,6 +47,38 @@ pub struct Histogram {
pub counts: Vec<u64>,
}
impl Histogram {
/// Return the bin index for a given value using the outlier-bracket layout.
#[cfg(test)]
pub fn bin_for_value(&self, value: f32) -> usize {
let num_bins = self.counts.len();
if value < self.p1 {
0
} else if value >= self.p99 {
num_bins - 1
} else {
let middle_bins = num_bins.saturating_sub(2);
if middle_bins > 0 && self.p99 > self.p1 {
let width = (self.p99 - self.p1) / middle_bins as f32;
let middle_bin = ((value - self.p1) / width) as usize;
(1 + middle_bin).min(num_bins - 2)
} else {
num_bins / 2
}
}
}
/// Width of a single middle bin (bins 1..n-2).
#[cfg(test)]
pub fn middle_bin_width(&self) -> f32 {
let middle_bins = self.counts.len().saturating_sub(2);
if middle_bins > 0 && self.p99 > self.p1 {
(self.p99 - self.p1) / middle_bins as f32
} else {
0.0
}
}
}
pub struct FeatureStats {
pub slider_min: f32,
@ -60,14 +92,67 @@ pub struct RenovationEvent {
pub event: String,
}
/// Lightweight reference to quantization parameters for decoding u16 feature data.
pub struct QuantRef<'a> {
pub dequant_a: &'a [f32],
pub quant_min: &'a [f32],
pub quant_range: &'a [f32],
pub num_numeric: usize,
}
impl QuantRef<'_> {
/// Decode a raw u16 value back to f32.
#[inline]
pub fn decode(&self, feat_idx: usize, raw: u16) -> f32 {
if raw == NAN_U16 {
return f32::NAN;
}
if feat_idx >= self.num_numeric {
raw as f32
} else {
raw as f32 * self.dequant_a[feat_idx] + self.quant_min[feat_idx]
}
}
/// Encode a filter minimum bound to u16 (floors to include boundary values).
#[inline]
pub fn encode_min(&self, feat_idx: usize, value: f32) -> u16 {
if !value.is_finite() || self.quant_range[feat_idx] == 0.0 {
return 0;
}
let norm = (value - self.quant_min[feat_idx]) / self.quant_range[feat_idx];
(norm * QUANT_SCALE).floor().clamp(0.0, QUANT_SCALE) as u16
}
/// Encode a filter maximum bound to u16 (ceils to include boundary values).
#[inline]
pub fn encode_max(&self, feat_idx: usize, value: f32) -> u16 {
if !value.is_finite() || self.quant_range[feat_idx] == 0.0 {
return QUANT_SCALE as u16;
}
let norm = (value - self.quant_min[feat_idx]) / self.quant_range[feat_idx];
(norm * QUANT_SCALE).ceil().clamp(0.0, QUANT_SCALE) as u16
}
}
pub struct PropertyData {
pub lat: Vec<f32>,
pub lon: Vec<f32>,
pub feature_names: Vec<String>,
pub num_features: usize,
/// Number of numeric features (enum features start at this index).
pub num_numeric: usize,
/// Row-major flat array: feature_data[row * num_features + feat_idx].
/// NaN = null. For enum features, stores the index as f32 (0.0, 1.0, etc).
pub feature_data: Vec<f32>,
/// Quantized to u16. NaN sentinel = u16::MAX (65535).
/// Numeric features: encoded via (val - min) / range * 65534.
/// Enum features: stored directly as u16 cast of the f32 index.
pub feature_data: Vec<u16>,
/// Per-feature: range / QUANT_SCALE for fast decode.
dequant_a: Vec<f32>,
/// Per-feature: minimum value (offset for dequantization).
quant_min: Vec<f32>,
/// Per-feature: max - min (for encoding filter bounds).
quant_range: Vec<f32>,
pub feature_stats: Vec<FeatureStats>,
/// Contiguous buffer holding all address strings end-to-end.
address_buffer: String,
@ -79,7 +164,7 @@ pub struct PropertyData {
postcode_interner: lasso::RodeoReader,
postcode_keys: Vec<lasso::Spur>,
/// For enum features: maps feature index to list of possible string values.
/// Index in values list corresponds to the f32 value stored in feature_data.
/// Index in values list corresponds to the u16 value stored in feature_data.
pub enum_values: rustc_hash::FxHashMap<usize, Vec<String>>,
/// Per-row flag: true = construction date is approximate (from EPC band),
/// false = exact (from new-build transaction date).
@ -91,10 +176,11 @@ pub struct PropertyData {
/// Per-row listing features (key feature bullet points from online listings).
/// Only rows with features are present in the map.
listing_features: FxHashMap<u32, Vec<String>>,
/// Per-row optional string columns from online listings.
listing_url: Vec<Option<String>>,
property_sub_type: Vec<Option<String>>,
price_qualifier: Vec<Option<String>>,
/// Sparse per-row optional string columns from online listings.
/// Only rows with non-empty values are stored (saves ~1 GB vs Vec<Option<String>>).
listing_url: FxHashMap<u32, String>,
property_sub_type: FxHashMap<u32, String>,
price_qualifier: FxHashMap<u32, String>,
}
impl PropertyData {
@ -139,17 +225,43 @@ impl PropertyData {
/// Get listing URL for a given row.
pub fn listing_url(&self, row: usize) -> Option<&str> {
self.listing_url[row].as_deref()
self.listing_url.get(&(row as u32)).map(String::as_str)
}
/// Get property sub-type for a given row.
pub fn property_sub_type(&self, row: usize) -> Option<&str> {
self.property_sub_type[row].as_deref()
self.property_sub_type
.get(&(row as u32))
.map(String::as_str)
}
/// Get price qualifier for a given row.
pub fn price_qualifier(&self, row: usize) -> Option<&str> {
self.price_qualifier[row].as_deref()
self.price_qualifier.get(&(row as u32)).map(String::as_str)
}
/// Decode a single feature value from quantized u16 storage.
#[inline]
pub fn get_feature(&self, row: usize, feat_idx: usize) -> f32 {
let raw = self.feature_data[row * self.num_features + feat_idx];
if raw == NAN_U16 {
return f32::NAN;
}
if feat_idx >= self.num_numeric {
raw as f32
} else {
raw as f32 * self.dequant_a[feat_idx] + self.quant_min[feat_idx]
}
}
/// Get a QuantRef for passing to aggregation/filter functions.
pub fn quant_ref(&self) -> QuantRef<'_> {
QuantRef {
dequant_a: &self.dequant_a,
quant_min: &self.quant_min,
quant_range: &self.quant_range,
num_numeric: self.num_numeric,
}
}
}
@ -355,13 +467,12 @@ pub fn precompute_h3(lat: &[f32], lon: &[f32]) -> anyhow::Result<Vec<u64>> {
.zip(lon.par_iter())
.enumerate()
.map(|(i, (&latitude, &longitude))| {
let coord = h3o::LatLng::new(latitude as f64, longitude as f64)
.unwrap_or_else(|err| {
panic!(
"Invalid coordinates at row {}: lat={}, lon={}: {}",
i, latitude, longitude, err
)
});
let coord = h3o::LatLng::new(latitude as f64, longitude as f64).unwrap_or_else(|err| {
panic!(
"Invalid coordinates at row {}: lat={}, lon={}: {}",
i, latitude, longitude, err
)
});
u64::from(coord.to_cell(h3_res))
})
.collect();
@ -378,7 +489,10 @@ impl PropertyData {
listings_rent_path: &Path,
) -> anyhow::Result<Self> {
// Load postcode.parquet
tracing::info!("Loading postcode features from {:?}", postcode_features_path);
tracing::info!(
"Loading postcode features from {:?}",
postcode_features_path
);
let postcode_df = LazyFrame::scan_parquet(postcode_features_path, Default::default())
.context("Failed to scan postcode parquet")?
.collect()
@ -623,6 +737,16 @@ impl PropertyData {
})
.collect::<anyhow::Result<Vec<_>>>()?;
// Compute quantization parameters from feature stats (numeric features)
let mut quant_min = Vec::with_capacity(num_features);
let mut quant_range = Vec::with_capacity(num_features);
for stats in &numeric_feature_stats {
let min = stats.histogram.min;
let max = stats.histogram.max;
quant_min.push(min);
quant_range.push(if max > min { max - min } else { 0.0 });
}
tracing::info!("Extracting string columns");
let extract_string_col = |df: &DataFrame, name: &str| -> anyhow::Result<Vec<String>> {
let column = df
@ -928,19 +1052,34 @@ impl PropertyData {
map
};
// Permute optional string columns
let listing_url: Vec<Option<String>> = perm
.iter()
.map(|&old_row| listing_url_raw[old_row as usize].clone())
.collect();
let property_sub_type: Vec<Option<String>> = perm
.iter()
.map(|&old_row| property_sub_type_raw[old_row as usize].clone())
.collect();
let price_qualifier: Vec<Option<String>> = perm
.iter()
.map(|&old_row| price_qualifier_raw[old_row as usize].clone())
.collect();
// Permute optional string columns into sparse HashMaps
let listing_url: FxHashMap<u32, String> = {
let mut map = FxHashMap::default();
for (new_row, &old_row) in perm.iter().enumerate() {
if let Some(ref s) = listing_url_raw[old_row as usize] {
map.insert(new_row as u32, s.clone());
}
}
map
};
let property_sub_type: FxHashMap<u32, String> = {
let mut map = FxHashMap::default();
for (new_row, &old_row) in perm.iter().enumerate() {
if let Some(ref s) = property_sub_type_raw[old_row as usize] {
map.insert(new_row as u32, s.clone());
}
}
map
};
let price_qualifier: FxHashMap<u32, String> = {
let mut map = FxHashMap::default();
for (new_row, &old_row) in perm.iter().enumerate() {
if let Some(ref s) = price_qualifier_raw[old_row as usize] {
map.insert(new_row as u32, s.clone());
}
}
map
};
// Build enum_values map: feature_index -> list of string values
let mut enum_values: rustc_hash::FxHashMap<usize, Vec<String>> =
@ -967,24 +1106,47 @@ impl PropertyData {
counts: vec![0; num_values.max(1)],
},
});
// Enum features: not quantized, stored directly as u16
quant_min.push(0.0);
quant_range.push(0.0);
}
let dequant_a: Vec<f32> = quant_range
.iter()
.map(|&r| if r > 0.0 { r / QUANT_SCALE } else { 0.0 })
.collect();
// Transpose to row-major AND apply spatial permutation in one pass.
// Combines numeric and enum features into a single feature_data array.
tracing::info!("Transposing to row-major layout (spatially sorted)");
let mut feature_data = vec![f32::NAN; row_count * num_features];
// Combines numeric and enum features into a single feature_data array, quantized to u16.
tracing::info!("Transposing to row-major layout (spatially sorted, quantized to u16)");
let mut feature_data = vec![NAN_U16; row_count * num_features];
feature_data
.par_chunks_mut(num_features)
.enumerate()
.for_each(|(new_row, row_slice)| {
let old_index = perm[new_row] as usize;
// Numeric features
// Numeric features: quantize to u16
for (feat_idx, col_vec) in numeric_col_major.iter().enumerate() {
row_slice[feat_idx] = col_vec[old_index];
let value = col_vec[old_index];
row_slice[feat_idx] = if value.is_finite() {
let range = quant_range[feat_idx];
if range > 0.0 {
let normalized = (value - quant_min[feat_idx]) / range;
(normalized * QUANT_SCALE).round().clamp(0.0, QUANT_SCALE) as u16
} else {
0
}
} else {
NAN_U16
};
}
// Enum features (stored as f32 indices)
// Enum features: store as u16 directly
for (enum_idx, (_, encoded)) in enum_col_major.iter().enumerate() {
row_slice[num_numeric + enum_idx] = encoded[old_index];
let value = encoded[old_index];
row_slice[num_numeric + enum_idx] = if value.is_finite() {
value as u16
} else {
NAN_U16
};
}
});
@ -995,7 +1157,11 @@ impl PropertyData {
lon,
feature_names,
num_features,
num_numeric,
feature_data,
dequant_a,
quant_min,
quant_range,
feature_stats,
address_buffer,
address_offsets,

21
server-rs/src/data/travel_time.rs
View file

@ -124,10 +124,7 @@ impl TravelTimeStore {
if file_name.ends_with(".parquet") {
let file_stem = file_name.trim_end_matches(".parquet");
let slug = strip_numeric_prefix(file_stem).to_string();
slug_to_file.insert(
(mode.clone(), slug.clone()),
file_stem.to_string(),
);
slug_to_file.insert((mode.clone(), slug.clone()), file_stem.to_string());
slugs.insert(slug);
}
}
@ -207,10 +204,7 @@ impl TravelTimeStore {
for (i, (pc, min)) in postcodes.into_iter().zip(minutes.into_iter()).enumerate() {
if let (Some(pc), Some(min)) = (pc, min) {
let best_min = best.as_ref().and_then(|b| b.get(i));
let journey = journeys
.as_ref()
.and_then(|j| j.get(i))
.map(Arc::from);
let journey = journeys.as_ref().and_then(|j| j.get(i)).map(Arc::from);
map.insert(
pc.to_string(),
TravelDataRow {
@ -274,10 +268,15 @@ mod tests {
#[test]
fn strip_numeric_prefix_basic() {
assert_eq!(strip_numeric_prefix("000000-bank-tube-station"), "bank-tube-station");
assert_eq!(
strip_numeric_prefix("000000-bank-tube-station"),
"bank-tube-station"
);
assert_eq!(strip_numeric_prefix("000123-abbey-hey"), "abbey-hey");
assert_eq!(strip_numeric_prefix("bank-tube-station"), "bank-tube-station");
assert_eq!(
strip_numeric_prefix("bank-tube-station"),
"bank-tube-station"
);
assert_eq!(strip_numeric_prefix("london"), "london");
}
}

1
server-rs/src/features.rs
View file

@ -62,7 +62,6 @@ pub struct EnumFeatureGroup {
pub features: &'static [EnumFeatureConfig],
}
pub static FEATURE_GROUPS: &[FeatureGroup] = &[
FeatureGroup {
name: "Properties in the area",

95
server-rs/src/main.rs
View file

@ -1,3 +1,5 @@
#![allow(clippy::min_ident_chars)]
mod aggregation;
mod auth;
mod consts;
@ -17,11 +19,11 @@ use std::sync::Arc;
use std::time::Duration;
use anyhow::{bail, Context};
use consts::SERVICE_CALL_TIMEOUT;
use axum::middleware;
use axum::routing::{any, get, patch, post};
use axum::Router;
use clap::Parser;
use consts::SERVICE_CALL_TIMEOUT;
use tower::limit::ConcurrencyLimitLayer;
use tower_http::compression::CompressionLayer;
@ -36,7 +38,10 @@ use tracing_subscriber::EnvFilter;
use state::AppState;
#[derive(Parser)]
#[command(name = "perfect-postcode", about = "Perfect Postcode property map server")]
#[command(
name = "perfect-postcode",
about = "Perfect Postcode property map server"
)]
struct Cli {
/// Path to properties.parquet (one row per historical property)
#[arg(long)]
@ -129,7 +134,6 @@ struct Cli {
/// Google OAuth client secret for PocketBase SSO
#[arg(long, env = "GOOGLE_OAUTH_CLIENT_SECRET")]
google_oauth_client_secret: String,
}
#[tokio::main]
@ -137,8 +141,7 @@ async fn main() -> anyhow::Result<()> {
let file_appender = tracing_appender::rolling::daily("logs", "server.log");
let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
let env_filter =
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
tracing_subscriber::registry()
.with(env_filter)
@ -332,10 +335,7 @@ async fn main() -> anyhow::Result<()> {
info!("Gemini configured (model: {})", cli.gemini_model);
let tt_path = &cli.travel_times;
if !tt_path.exists() {
bail!(
"Travel times directory not found: {}",
tt_path.display()
);
bail!("Travel times directory not found: {}", tt_path.display());
}
info!("Loading travel time data from {}", tt_path.display());
let travel_time_store = {
@ -476,7 +476,9 @@ async fn main() -> anyhow::Result<()> {
)
.route(
"/api/travel-destinations",
get(move |query| routes::get_travel_destinations(state_travel_destinations.clone(), query)),
get(move |query| {
routes::get_travel_destinations(state_travel_destinations.clone(), query)
}),
)
.route(
"/api/journey",
@ -490,24 +492,34 @@ async fn main() -> anyhow::Result<()> {
)
.route(
"/api/hexagon-stats",
get(move |ext, query| routes::get_hexagon_stats(state_hexagon_stats.clone(), ext, query)),
get(move |ext, query| {
routes::get_hexagon_stats(state_hexagon_stats.clone(), ext, query)
}),
)
.route(
"/api/postcode-stats",
get(move |ext, query| routes::get_postcode_stats(state_postcode_stats.clone(), ext, query)),
get(move |ext, query| {
routes::get_postcode_stats(state_postcode_stats.clone(), ext, query)
}),
)
.route(
"/api/postcode-properties",
get(move |ext, query| routes::get_postcode_properties(state_postcode_properties.clone(), ext, query)),
get(move |ext, query| {
routes::get_postcode_properties(state_postcode_properties.clone(), ext, query)
}),
)
.route(
"/api/screenshot",
get(move |headers, query| routes::get_screenshot(state_screenshot.clone(), headers, query)),
get(move |headers, query| {
routes::get_screenshot(state_screenshot.clone(), headers, query)
}),
)
.route(
"/api/export",
get(move |headers, ext, query| routes::get_export(state_export.clone(), headers, ext, query))
.layer(ConcurrencyLimitLayer::new(3)),
get(move |headers, ext, query| {
routes::get_export(state_export.clone(), headers, ext, query)
})
.layer(ConcurrencyLimitLayer::new(3)),
)
.route("/api/me", get(routes::get_me))
.route(
@ -525,9 +537,7 @@ async fn main() -> anyhow::Result<()> {
)
.route(
"/api/newsletter",
patch(move |ext, body| {
routes::patch_newsletter(state_newsletter.clone(), ext, body)
}),
patch(move |ext, body| routes::patch_newsletter(state_newsletter.clone(), ext, body)),
)
.route(
"/api/pricing",
@ -546,8 +556,9 @@ async fn main() -> anyhow::Result<()> {
)
.route(
"/api/invites",
get(move |ext| routes::get_invites(state_invites_list.clone(), ext))
.post(move |ext, body| routes::post_invites(state_invites_create.clone(), ext, body)),
get(move |ext| routes::get_invites(state_invites_list.clone(), ext)).post(
move |ext, body| routes::post_invites(state_invites_create.clone(), ext, body),
),
)
.route(
"/api/invite/{code}",
@ -591,35 +602,35 @@ async fn main() -> anyhow::Result<()> {
);
let app = if let Some(ref dist) = cli.dist {
api.fallback_service(
ServeDir::new(dist).fallback(ServeFile::new(dist.join("index.html"))),
)
api.fallback_service(ServeDir::new(dist).fallback(ServeFile::new(dist.join("index.html"))))
} else {
api
}
.layer(middleware::from_fn(metrics::track_metrics))
.layer(middleware::from_fn(auth::auth_middleware))
.layer(middleware::from_fn(
move |req: axum::extract::Request, next: middleware::Next| {
let st = state_crawler.clone();
async move {
// Inject state into request extensions for auth + OG middleware
let (mut parts, body) = req.into_parts();
parts.extensions.insert(st);
let req = axum::extract::Request::from_parts(parts, body);
og_middleware::og_middleware(req, next).await
}
},
))
.layer(cors)
.layer(CompressionLayer::new().zstd(true).gzip(true))
.layer(TraceLayer::new_for_http());
.layer(middleware::from_fn(metrics::track_metrics))
.layer(middleware::from_fn(auth::auth_middleware))
.layer(middleware::from_fn(
move |req: axum::extract::Request, next: middleware::Next| {
let st = state_crawler.clone();
async move {
// Inject state into request extensions for auth + OG middleware
let (mut parts, body) = req.into_parts();
parts.extensions.insert(st);
let req = axum::extract::Request::from_parts(parts, body);
og_middleware::og_middleware(req, next).await
}
},
))
.layer(cors)
.layer(CompressionLayer::new().zstd(true).gzip(true))
.layer(TraceLayer::new_for_http());
// Lock all current and future memory pages to prevent swapping
unsafe {
if libc::mlockall(libc::MCL_CURRENT | libc::MCL_FUTURE) != 0 {
let err = std::io::Error::last_os_error();
tracing::warn!("mlockall failed (need CAP_IPC_LOCK or sufficient RLIMIT_MEMLOCK): {err}");
tracing::warn!(
"mlockall failed (need CAP_IPC_LOCK or sufficient RLIMIT_MEMLOCK): {err}"
);
} else {
info!("All memory pages locked (mlockall)");
}

8
server-rs/src/og_middleware.rs
View file

@ -8,7 +8,8 @@ use axum::response::Response;
use crate::state::AppState;
const OG_PLACEHOLDER: &str = r#"<meta name="x-og-placeholder" content="__PERFECT_POSTCODE_OG_TAGS__"/>"#;
const OG_PLACEHOLDER: &str =
r#"<meta name="x-og-placeholder" content="__PERFECT_POSTCODE_OG_TAGS__"/>"#;
pub async fn og_middleware(request: Request, next: Next) -> Response {
let path = request.uri().path().to_string();
@ -51,10 +52,7 @@ pub async fn og_middleware(request: Request, next: Next) -> Response {
let og_image_url = if is_invite {
// Include path= so the screenshot service navigates to /invite/CODE
if query_string.is_empty() {
format!(
"{}/api/screenshot?og=1&path={}",
state.public_url, path
)
format!("{}/api/screenshot?og=1&path={}", state.public_url, path)
} else {
format!(
"{}/api/screenshot?og=1&path={}&{}",

7
server-rs/src/parsing/fields.rs
View file

@ -24,12 +24,7 @@ pub fn parse_field_indices(
}
match name_to_index.get(name) {
Some(&idx) => indices.push(idx),
None => {
return Err((
StatusCode::BAD_REQUEST,
format!("Unknown field: {}", name),
))
}
None => return Err((StatusCode::BAD_REQUEST, format!("Unknown field: {}", name))),
}
}
Ok(Some(indices))

226
server-rs/src/parsing/filters.rs
View file

@ -1,20 +1,22 @@
use rustc_hash::{FxHashMap, FxHashSet};
/// Filter for numeric features: value must be in [min, max] range.
use crate::consts::NAN_U16;
use crate::data::QuantRef;
/// Filter for numeric features: value must be in [min_u16, max_u16] range (quantized).
#[derive(Debug)]
pub struct ParsedFilter {
pub feat_idx: usize,
pub min: f32,
pub max: f32,
pub min_u16: u16,
pub max_u16: u16,
}
/// Filter for enum features: value must be one of the allowed indices.
/// Uses FxHashSet<u32> (f32 bits) for O(1) lookups instead of O(n) Vec::contains.
/// Filter for enum features: value must be one of the allowed u16 indices.
#[derive(Debug)]
pub struct ParsedEnumFilter {
pub feat_idx: usize,
/// Allowed enum indices stored as f32 bits for exact comparison
pub allowed: FxHashSet<u32>,
/// Allowed enum indices as u16.
pub allowed: FxHashSet<u16>,
}
/// Parse `;;`-separated filter string into numeric and enum filters.
@ -26,6 +28,7 @@ pub fn parse_filters(
filter_str: Option<&str>,
feature_name_to_index: &FxHashMap<String, usize>,
enum_values: &FxHashMap<usize, Vec<String>>,
quant: &QuantRef,
) -> Result<(Vec<ParsedFilter>, Vec<ParsedEnumFilter>), String> {
let mut numeric = Vec::new();
let mut enums = Vec::new();
@ -50,20 +53,20 @@ pub fn parse_filters(
// Check if this is an enum feature
if let Some(values) = enum_values.get(&feat_idx) {
// Enum filter: convert string values to f32 indices (stored as bits for O(1) lookup)
let allowed: FxHashSet<u32> = rest
// Enum filter: convert string values to u16 indices
let allowed: FxHashSet<u16> = rest
.split('|')
.filter_map(|value| {
let value = value.trim();
values
.iter()
.position(|existing| existing == value)
.map(|position| (position as f32).to_bits())
.map(|position| position as u16)
})
.collect();
enums.push(ParsedEnumFilter { feat_idx, allowed });
} else {
// Numeric filter: parse min:max
// Numeric filter: parse min:max and encode to u16
let num_parts: Vec<&str> = rest.splitn(2, ':').collect();
if num_parts.len() != 2 {
return Err(format!(
@ -78,7 +81,11 @@ pub fn parse_filters(
.trim()
.parse::<f32>()
.map_err(|err| format!("Invalid max value in filter '{name}': {err}"))?;
numeric.push(ParsedFilter { feat_idx, min, max });
numeric.push(ParsedFilter {
feat_idx,
min_u16: quant.encode_min(feat_idx, min),
max_u16: quant.encode_max(feat_idx, max),
});
}
}
@ -86,23 +93,22 @@ pub fn parse_filters(
}
/// Check if a row passes all filters.
/// All features (numeric and enum) are stored in feature_data as f32.
/// All features (numeric and enum) are stored in feature_data as quantized u16.
pub fn row_passes_filters(
row: usize,
filters: &[ParsedFilter],
enum_filters: &[ParsedEnumFilter],
feature_data: &[f32],
feature_data: &[u16],
num_features: usize,
) -> bool {
let base = row * num_features;
filters.iter().all(|filter| {
let value = feature_data[base + filter.feat_idx];
value.is_finite() && value >= filter.min && value <= filter.max
let raw = feature_data[base + filter.feat_idx];
raw != NAN_U16 && raw >= filter.min_u16 && raw <= filter.max_u16
}) && enum_filters.iter().all(|filter| {
let value = feature_data[base + filter.feat_idx];
// O(1) lookup using f32 bits as key
value.is_finite() && filter.allowed.contains(&value.to_bits())
let raw = feature_data[base + filter.feat_idx];
raw != NAN_U16 && filter.allowed.contains(&raw)
})
}
@ -151,52 +157,127 @@ mod tests {
map
}
fn allowed_set(values: &[f32]) -> FxHashSet<u32> {
values.iter().map(|v| v.to_bits()).collect()
/// Build a test QuantRef with known parameters.
/// num_numeric indicates how many of the features are numeric (the rest are enum).
fn test_quant(num_features: usize, num_numeric: usize) -> TestQuant {
// For numeric features: use min=0, range=1000 (simple mapping)
let mut quant_min = vec![0.0f32; num_features];
let mut quant_range = vec![1000.0f32; num_features];
let mut dequant_a = vec![1000.0 / crate::consts::QUANT_SCALE; num_features];
// Enum features: no quantization
for i in num_numeric..num_features {
quant_min[i] = 0.0;
quant_range[i] = 0.0;
dequant_a[i] = 0.0;
}
TestQuant {
dequant_a,
quant_min,
quant_range,
num_numeric,
}
}
struct TestQuant {
dequant_a: Vec<f32>,
quant_min: Vec<f32>,
quant_range: Vec<f32>,
num_numeric: usize,
}
impl TestQuant {
fn as_ref(&self) -> QuantRef<'_> {
QuantRef {
dequant_a: &self.dequant_a,
quant_min: &self.quant_min,
quant_range: &self.quant_range,
num_numeric: self.num_numeric,
}
}
/// Encode a f32 value to u16 using the test quantization parameters.
fn encode(&self, feat_idx: usize, value: f32) -> u16 {
if !value.is_finite() {
return NAN_U16;
}
if feat_idx >= self.num_numeric {
// Enum: store directly
return value as u16;
}
let range = self.quant_range[feat_idx];
if range > 0.0 {
let normalized = (value - self.quant_min[feat_idx]) / range;
(normalized * crate::consts::QUANT_SCALE)
.round()
.clamp(0.0, crate::consts::QUANT_SCALE) as u16
} else {
0
}
}
}
#[test]
fn parse_filters_numeric() {
let tq = test_quant(3, 2);
let (numeric, enums) = parse_filters(
Some("price:100:500"),
&feature_name_to_index(),
&enum_values(),
&tq.as_ref(),
)
.unwrap();
assert_eq!(numeric.len(), 1);
assert_eq!(numeric[0].feat_idx, 0);
assert_eq!(numeric[0].min, 100.0);
assert_eq!(numeric[0].max, 500.0);
// min_u16 should be floor(100/1000 * 65534) = floor(6553.4) = 6553
assert_eq!(numeric[0].min_u16, 6553);
// max_u16 should be ceil(500/1000 * 65534) = ceil(32767.0) = 32767
assert_eq!(numeric[0].max_u16, 32767);
assert!(enums.is_empty());
}
#[test]
fn parse_filters_enum() {
let (numeric, enums) =
parse_filters(Some("rating:A|C"), &feature_name_to_index(), &enum_values()).unwrap();
let tq = test_quant(3, 2);
let (numeric, enums) = parse_filters(
Some("rating:A|C"),
&feature_name_to_index(),
&enum_values(),
&tq.as_ref(),
)
.unwrap();
assert!(numeric.is_empty());
assert_eq!(enums.len(), 1);
assert_eq!(enums[0].feat_idx, 2);
assert!(enums[0].allowed.contains(&(0.0_f32).to_bits())); // A = index 0
assert!(enums[0].allowed.contains(&(2.0_f32).to_bits())); // C = index 2
assert!(enums[0].allowed.contains(&0)); // A = index 0
assert!(enums[0].allowed.contains(&2)); // C = index 2
assert_eq!(enums[0].allowed.len(), 2);
}
#[test]
fn parse_filters_empty() {
let (n, e) = parse_filters(None, &feature_name_to_index(), &enum_values()).unwrap();
let tq = test_quant(3, 2);
let (n, e) =
parse_filters(None, &feature_name_to_index(), &enum_values(), &tq.as_ref()).unwrap();
assert!(n.is_empty() && e.is_empty());
let (n, e) = parse_filters(Some(""), &feature_name_to_index(), &enum_values()).unwrap();
let (n, e) = parse_filters(
Some(""),
&feature_name_to_index(),
&enum_values(),
&tq.as_ref(),
)
.unwrap();
assert!(n.is_empty() && e.is_empty());
}
#[test]
fn parse_filters_unknown_feature_errors() {
let tq = test_quant(3, 2);
let result = parse_filters(
Some("unknown:1:2"),
&feature_name_to_index(),
&enum_values(),
&tq.as_ref(),
);
assert!(result.is_err());
assert!(result.unwrap_err().contains("Unknown feature"));
@ -204,12 +285,13 @@ mod tests {
#[test]
fn row_passes_numeric_filter() {
let tq = test_quant(1, 1);
let filters = vec![ParsedFilter {
feat_idx: 0,
min: 10.0,
max: 20.0,
min_u16: tq.as_ref().encode_min(0, 10.0),
max_u16: tq.as_ref().encode_max(0, 20.0),
}];
let data = vec![15.0, 5.0, f32::NAN];
let data = vec![tq.encode(0, 15.0), tq.encode(0, 5.0), NAN_U16];
assert!(row_passes_filters(0, &filters, &[], &data, 1));
assert!(!row_passes_filters(1, &filters, &[], &data, 1));
@ -218,12 +300,18 @@ mod tests {
#[test]
fn row_passes_enum_filter() {
let allowed: FxHashSet<u32> = [0.0_f32, 2.0].iter().map(|v| v.to_bits()).collect();
let tq = test_quant(1, 0); // all enum
let allowed: FxHashSet<u16> = [0u16, 2].into_iter().collect();
let filters = vec![ParsedEnumFilter {
feat_idx: 0,
allowed,
}];
let data = vec![0.0, 1.0, 2.0, f32::NAN];
let data = vec![
tq.encode(0, 0.0),
tq.encode(0, 1.0),
tq.encode(0, 2.0),
NAN_U16,
];
assert!(row_passes_filters(0, &[], &filters, &data, 1));
assert!(!row_passes_filters(1, &[], &filters, &data, 1));
@ -233,10 +321,12 @@ mod tests {
#[test]
fn parse_multiple_numeric_filters() {
let tq = test_quant(4, 2);
let (numeric, _enums) = parse_filters(
Some("Price:100000:500000;;Area:50:200"),
&extended_feature_map(),
&extended_enum_values(),
&tq.as_ref(),
)
.unwrap();
@ -247,10 +337,12 @@ mod tests {
#[test]
fn parse_mixed_filters() {
let tq = test_quant(4, 2);
let (numeric, enums) = parse_filters(
Some("Price:100000:500000;;Type:Semi|Terraced"),
&extended_feature_map(),
&extended_enum_values(),
&tq.as_ref(),
)
.unwrap();
@ -260,10 +352,12 @@ mod tests {
#[test]
fn parse_invalid_numeric_format_errors() {
let tq = test_quant(4, 2);
let result = parse_filters(
Some("Price:not_a_number:500000"),
&extended_feature_map(),
&extended_enum_values(),
&tq.as_ref(),
);
assert!(result.is_err());
@ -272,25 +366,29 @@ mod tests {
#[test]
fn parse_enum_with_unknown_value() {
let tq = test_quant(4, 2);
let (_numeric, enums) = parse_filters(
Some("Type:Detached|Unknown|Flats/Maisonettes"),
&extended_feature_map(),
&extended_enum_values(),
&tq.as_ref(),
)
.unwrap();
assert_eq!(enums.len(), 1);
assert!(enums[0].allowed.contains(&(0.0_f32).to_bits())); // Detached
assert!(enums[0].allowed.contains(&(3.0_f32).to_bits())); // Flats/Maisonettes
assert!(enums[0].allowed.contains(&0)); // Detached
assert!(enums[0].allowed.contains(&3)); // Flats/Maisonettes
assert_eq!(enums[0].allowed.len(), 2);
}
#[test]
fn parse_filter_with_whitespace() {
let tq = test_quant(4, 2);
let (numeric, enums) = parse_filters(
Some("Price : 100000 : 500000 ;; Type : Detached | Flats/Maisonettes"),
&extended_feature_map(),
&extended_enum_values(),
&tq.as_ref(),
)
.unwrap();
@ -300,25 +398,30 @@ mod tests {
#[test]
fn row_passes_no_filters() {
let feature_data = vec![100.0_f32, 50.0];
let tq = test_quant(2, 2);
let feature_data = vec![tq.encode(0, 100.0), tq.encode(1, 50.0)];
assert!(row_passes_filters(0, &[], &[], &feature_data, 2));
}
#[test]
fn row_passes_numeric_filter_at_boundary() {
let tq = test_quant(1, 1);
let filters = vec![ParsedFilter {
feat_idx: 0,
min: 100.0,
max: 200.0,
min_u16: tq.as_ref().encode_min(0, 100.0),
max_u16: tq.as_ref().encode_max(0, 200.0),
}];
assert!(row_passes_filters(0, &filters, &[], &[100.0], 1));
assert!(row_passes_filters(0, &filters, &[], &[200.0], 1));
let data_100 = vec![tq.encode(0, 100.0)];
let data_200 = vec![tq.encode(0, 200.0)];
assert!(row_passes_filters(0, &filters, &[], &data_100, 1));
assert!(row_passes_filters(0, &filters, &[], &data_200, 1));
}
#[test]
fn row_fails_empty_enum_filter() {
let feature_data = vec![1.0_f32];
let tq = test_quant(1, 0);
let feature_data = vec![tq.encode(0, 1.0)];
let enum_filters = vec![ParsedEnumFilter {
feat_idx: 0,
allowed: FxHashSet::default(),
@ -328,16 +431,17 @@ mod tests {
#[test]
fn multiple_filters_all_must_pass() {
let feature_data = vec![150.0_f32, 1.0];
let tq = test_quant(2, 1); // feat 0 = numeric, feat 1 = enum
let feature_data = vec![tq.encode(0, 150.0), tq.encode(1, 1.0)];
let numeric_filters = vec![ParsedFilter {
feat_idx: 0,
min: 100.0,
max: 200.0,
min_u16: tq.as_ref().encode_min(0, 100.0),
max_u16: tq.as_ref().encode_max(0, 200.0),
}];
let enum_filters = vec![ParsedEnumFilter {
feat_idx: 1,
allowed: allowed_set(&[1.0, 2.0]),
allowed: [1u16, 2].into_iter().collect(),
}];
assert!(row_passes_filters(
@ -350,7 +454,7 @@ mod tests {
let enum_filters_fail = vec![ParsedEnumFilter {
feat_idx: 1,
allowed: allowed_set(&[0.0, 2.0]),
allowed: [0u16, 2].into_iter().collect(),
}];
assert!(!row_passes_filters(
0,
@ -363,17 +467,21 @@ mod tests {
#[test]
fn row_major_layout_correct_indexing() {
let tq = test_quant(2, 1); // feat 0 = numeric, feat 1 = enum
let feature_data = vec![
100.0_f32, 0.0, // Row 0
200.0, 1.0, // Row 1
300.0, 2.0, // Row 2
tq.encode(0, 100.0),
tq.encode(1, 0.0), // Row 0
tq.encode(0, 200.0),
tq.encode(1, 1.0), // Row 1
tq.encode(0, 300.0),
tq.encode(1, 2.0), // Row 2
];
let num_features = 2;
let filters = vec![ParsedFilter {
feat_idx: 0,
min: 150.0,
max: 250.0,
min_u16: tq.as_ref().encode_min(0, 150.0),
max_u16: tq.as_ref().encode_max(0, 250.0),
}];
assert!(!row_passes_filters(
@ -401,22 +509,24 @@ mod tests {
#[test]
fn filter_at_float_precision_boundary() {
let value = 100.0_f32;
let tq = test_quant(1, 1);
let value = tq.encode(0, 100.0);
let filter = ParsedFilter {
feat_idx: 0,
min: 100.0 - f32::EPSILON,
max: 100.0 + f32::EPSILON,
min_u16: tq.as_ref().encode_min(0, 100.0 - f32::EPSILON),
max_u16: tq.as_ref().encode_max(0, 100.0 + f32::EPSILON),
};
assert!(row_passes_filters(0, &[filter], &[], &[value], 1));
}
#[test]
fn enum_filter_with_fractional_index() {
let feature_data = vec![1.5_f32]; // Not exactly 1.0 or 2.0
fn enum_filter_rejects_non_matching_index() {
let tq = test_quant(1, 0); // all enum
let feature_data = vec![tq.encode(0, 3.0)]; // index 3
let enum_filters = vec![ParsedEnumFilter {
feat_idx: 0,
allowed: allowed_set(&[1.0, 2.0]),
allowed: [1u16, 2].into_iter().collect(),
}];
assert!(!row_passes_filters(0, &[], &enum_filters, &feature_data, 1));

5
server-rs/src/parsing/h3.rs
View file

@ -39,7 +39,10 @@ pub fn cell_for_row(
return max_cell;
}
let cell = h3o::CellIndex::try_from(max_cell).expect("precomputed H3 cell must be valid");
u64::from(cell.parent(h3_res).expect("parent resolution must be valid for precomputed cell"))
u64::from(
cell.parent(h3_res)
.expect("parent resolution must be valid for precomputed cell"),
)
}
/// Whether the given resolution requires computing a parent from precomputed cells.

13
server-rs/src/pocketbase.rs
View file

@ -214,11 +214,7 @@ async fn find_users_collection_id(
/// Ensure `is_admin` (bool) and `subscription` (text) fields exist on the `users` collection.
/// PocketBase PATCH replaces the entire `fields` array, so we must preserve existing fields.
async fn ensure_user_fields(
client: &Client,
base_url: &str,
token: &str,
) -> anyhow::Result<()> {
async fn ensure_user_fields(client: &Client, base_url: &str, token: &str) -> anyhow::Result<()> {
let url = format!("{base_url}/api/collections/users");
let resp = client
.get(&url)
@ -243,7 +239,12 @@ async fn ensure_user_fields(
let has_ai_tokens_used = fields.iter().any(|f| f["name"] == "ai_tokens_used");
let has_ai_tokens_week = fields.iter().any(|f| f["name"] == "ai_tokens_week");
if has_is_admin && has_subscription && has_newsletter && has_ai_tokens_used && has_ai_tokens_week {
if has_is_admin
&& has_subscription
&& has_newsletter
&& has_ai_tokens_used
&& has_ai_tokens_week
{
info!("PocketBase users collection already has all required fields");
return Ok(());
}

14
server-rs/src/routes.rs
View file

@ -7,24 +7,24 @@ pub(crate) mod hexagons;
mod invites;
mod journey;
mod me;
mod newsletter;
mod pb_proxy;
mod places;
mod pois;
mod postcode_properties;
mod postcode_stats;
mod postcodes;
pub(crate) mod pricing;
pub(crate) mod properties;
mod screenshot;
mod shorten;
mod stats;
mod streetview;
mod stripe_webhook;
mod newsletter;
pub(crate) mod pricing;
mod tiles;
pub(crate) mod travel_time;
mod travel_destinations;
mod travel_modes;
pub(crate) mod travel_time;
pub use ai_filters::{build_system_prompt, post_ai_filters};
pub use checkout::post_checkout;
@ -32,21 +32,21 @@ pub use export::get_export;
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
pub use hexagon_stats::get_hexagon_stats;
pub use hexagons::get_hexagons;
pub use invites::{get_invite, get_invites, post_invites, post_redeem_invite};
pub use journey::get_journey;
pub use me::get_me;
pub use newsletter::patch_newsletter;
pub use pb_proxy::proxy_to_pocketbase;
pub use places::get_places;
pub use pois::{get_poi_categories, get_pois};
pub use postcode_properties::get_postcode_properties;
pub use postcode_stats::get_postcode_stats;
pub use postcodes::{get_postcode_lookup, get_postcodes};
pub use pricing::get_pricing;
pub use properties::get_hexagon_properties;
pub use screenshot::{fetch_screenshot_bytes, get_screenshot};
pub use shorten::{get_short_url, post_shorten};
pub use streetview::get_streetview;
pub use invites::{get_invite, get_invites, post_invites, post_redeem_invite};
pub use journey::get_journey;
pub use newsletter::patch_newsletter;
pub use pricing::get_pricing;
pub use stripe_webhook::post_stripe_webhook;
pub use tiles::{get_style, get_tile, init_tile_reader};
pub use travel_destinations::get_travel_destinations;

42
server-rs/src/routes/ai_filters.rs
View file

@ -125,7 +125,9 @@ fn execute_destination_search(state: &AppState, query: &str, mode: &str) -> Valu
let slug_set = match tt_store.destinations.get(mode) {
Some(slugs) => slugs,
None => return json!({ "results": [], "message": format!("No travel data available for mode '{}'", mode) }),
None => {
return json!({ "results": [], "message": format!("No travel data available for mode '{}'", mode) })
}
};
// Find places matching the query that have travel time data.
@ -154,7 +156,11 @@ fn execute_destination_search(state: &AppState, query: &str, mode: &str) -> Valu
matches.truncate(10);
if matches.is_empty() {
info!(query = query, mode = mode, "Destination search returned no results");
info!(
query = query,
mode = mode,
"Destination search returned no results"
);
return json!({
"results": [],
"message": format!("No travel time data available for '{}' by {}. This destination cannot be used as a travel time filter.", query, mode)
@ -597,10 +603,7 @@ pub async fn post_ai_filters(
.and_then(|c| c.get("content"))
.ok_or_else(|| {
warn!("Malformed Gemini response: missing candidates[0].content");
(
StatusCode::BAD_GATEWAY,
"Malformed Gemini response".into(),
)
(StatusCode::BAD_GATEWAY, "Malformed Gemini response".into())
})?;
let parts = candidate
@ -608,10 +611,7 @@ pub async fn post_ai_filters(
.and_then(|p| p.as_array())
.ok_or_else(|| {
warn!("Malformed Gemini response: missing parts array");
(
StatusCode::BAD_GATEWAY,
"Malformed Gemini response".into(),
)
(StatusCode::BAD_GATEWAY, "Malformed Gemini response".into())
})?;
// Check if the model made a function call.
@ -621,17 +621,10 @@ pub async fn post_ai_filters(
let fn_name = fc.get("name").and_then(|n| n.as_str()).unwrap_or("");
let fn_args = fc.get("args").cloned().unwrap_or(json!({}));
info!(
function = fn_name,
round = round,
"AI called tool"
);
info!(function = fn_name, round = round, "AI called tool");
let fn_result = if fn_name == "search_destinations" {
let query = fn_args
.get("query")
.and_then(|q| q.as_str())
.unwrap_or("");
let query = fn_args.get("query").and_then(|q| q.as_str()).unwrap_or("");
let mode = fn_args
.get("mode")
.and_then(|m| m.as_str())
@ -710,7 +703,10 @@ pub async fn post_ai_filters(
}
// Exhausted tool rounds without getting a final text response
warn!("AI exhausted {} tool-calling rounds without final response", MAX_TOOL_ROUNDS);
warn!(
"AI exhausted {} tool-calling rounds without final response",
MAX_TOOL_ROUNDS
);
Err((
StatusCode::BAD_GATEWAY,
"AI could not complete the request".into(),
@ -746,7 +742,11 @@ fn validate_travel_time_filters(raw: &Value, state: &AppState) -> Vec<TravelTime
// Verify this destination actually exists
if !tt_store.has_destination(mode, slug) {
warn!(mode = mode, slug = slug, "AI suggested non-existent destination");
warn!(
mode = mode,
slug = slug,
"AI suggested non-existent destination"
);
continue;
}

18
server-rs/src/routes/checkout.rs
View file

@ -81,7 +81,10 @@ pub async fn post_checkout(
// If a referral code is provided and valid, look it up and apply the coupon
if let Some(ref code) = req.referral_code {
if validate_referral_invite(&state, code).await {
form_params.push(("discounts[0][coupon]", state.stripe_referral_coupon_id.clone()));
form_params.push((
"discounts[0][coupon]",
state.stripe_referral_coupon_id.clone(),
));
info!(code = %code, "Applying referral coupon to checkout");
}
}
@ -127,7 +130,13 @@ pub async fn post_checkout(
/// Grant a license by updating the user's subscription to "licensed" in PocketBase.
async fn grant_license(state: &AppState, user_id: &str) -> anyhow::Result<()> {
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await?;
let token = auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await?;
let url = format!("{pb_url}/api/collections/users/records/{user_id}");
let resp = state
@ -151,10 +160,7 @@ async fn grant_license(state: &AppState, user_id: &str) -> anyhow::Result<()> {
/// Check if a referral invite code exists and is unused.
async fn validate_referral_invite(state: &AppState, code: &str) -> bool {
// Only allow alphanumeric codes to prevent PocketBase filter injection
if code.is_empty()
|| code.len() > 20
|| !code.bytes().all(|b| b.is_ascii_alphanumeric())
{
if code.is_empty() || code.len() > 20 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) {
return false;
}

33
server-rs/src/routes/export.rs
View file

@ -12,6 +12,8 @@ use serde::Deserialize;
use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::NAN_U16;
use crate::data::QuantRef;
use crate::licensing::check_license_bounds;
use crate::parsing::{parse_field_indices, parse_filters, require_bounds, row_passes_filters};
use crate::routes::{fetch_screenshot_bytes, FeatureInfo};
@ -50,18 +52,20 @@ impl PostcodeExportAgg {
#[inline]
fn add_row(
&mut self,
feature_data: &[f32],
feature_data: &[u16],
row: usize,
num_features: usize,
enum_indices: &FxHashMap<usize, ()>,
quant: &QuantRef,
) {
self.count += 1;
let base = row * num_features;
let row_slice = &feature_data[base..base + num_features];
for (feat_idx, &value) in row_slice.iter().enumerate() {
if !value.is_finite() {
for (feat_idx, &raw) in row_slice.iter().enumerate() {
if raw == NAN_U16 {
continue;
}
let value = quant.decode(feat_idx, raw);
if enum_indices.contains_key(&feat_idx) {
*self
.enum_freqs
@ -131,10 +135,12 @@ pub async fn get_export(
check_license_bounds(&user.0, (south, west, north, east))?;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let filters_str = params.filters;
@ -188,6 +194,7 @@ pub async fn get_export(
let t0 = std::time::Instant::now();
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let quant = state.data.quant_ref();
let feature_names = &state.data.feature_names;
let enum_values = &state.data.enum_values;
let postcode_data = &state.postcode_data;
@ -222,7 +229,7 @@ pub async fn get_export(
for (pc_idx, rows) in postcode_rows {
let mut agg = PostcodeExportAgg::new(num_features);
for &row in &rows {
agg.add_row(feature_data, row, num_features, &enum_indices);
agg.add_row(feature_data, row, num_features, &enum_indices, &quant);
}
if agg.count > 0 {
postcode_aggs.push((pc_idx, agg));
@ -470,11 +477,9 @@ pub async fn get_export(
let mode_idx = mode_f32 as usize;
if let Some(values) = enum_values.get(&feat_idx) {
if mode_idx < values.len() {
sheet
.write_string(row, col, &values[mode_idx])
.map_err(|e| {
format!("Failed to write enum value: {e}")
})?;
sheet.write_string(row, col, &values[mode_idx]).map_err(
|e| format!("Failed to write enum value: {e}"),
)?;
}
}
}
@ -486,13 +491,11 @@ pub async fn get_export(
if let Some(fmt) = feat_num_fmts.get(&feat_idx) {
sheet
.write_number_with_format(row, col, mean, fmt)
.map_err(|e| {
format!("Failed to write numeric value: {e}")
})?;
.map_err(|e| format!("Failed to write numeric value: {e}"))?;
} else {
sheet.write_number(row, col, mean).map_err(|e| {
format!("Failed to write numeric value: {e}")
})?;
sheet
.write_number(row, col, mean)
.map_err(|e| format!("Failed to write numeric value: {e}"))?;
}
}
}

18
server-rs/src/routes/hexagon_stats.rs
View file

@ -100,10 +100,12 @@ pub async fn get_hexagon_stats(
check_license_bounds(&user.0, h3_bounds)?;
let h3_str = params.h3;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -114,10 +116,7 @@ pub async fn get_hexagon_stats(
// Load travel time data for central_postcode selection (if requested)
let journey_travel_data = match (&params.journey_mode, &params.journey_slug) {
(Some(mode), Some(slug)) if state.travel_time_store.has_destination(mode, slug) => {
state
.travel_time_store
.get(mode, slug)
.ok()
state.travel_time_store.get(mode, slug).ok()
}
_ => None,
};
@ -209,18 +208,13 @@ pub async fn get_hexagon_stats(
None
};
let price_history = stats::extract_price_history(
&matching_rows,
feature_data,
num_features,
&state.feature_name_to_index,
);
let price_history =
stats::extract_price_history(&matching_rows, &state.data, &state.feature_name_to_index);
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
&matching_rows,
feature_data,
&state.data,
&state.data.feature_names,
num_features,
&state.data.enum_values,
&state.data.feature_stats,
fields_specified,

19
server-rs/src/routes/hexagons.rs
View file

@ -40,7 +40,6 @@ pub struct HexagonParams {
travel: Option<String>,
}
/// Build feature maps from aggregated cell data, filtering to only cells that intersect the query bounds.
#[allow(clippy::too_many_arguments)]
fn build_feature_maps(
@ -144,10 +143,12 @@ pub async fn get_hexagons(
check_license_bounds(&user.0, (south, west, north, east))?;
}
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -185,6 +186,7 @@ pub async fn get_hexagons(
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let quant = state.data.quant_ref();
let (pc_interner, pc_keys) = state.data.postcode_parts();
let min_keys = &state.min_keys;
let max_keys = &state.max_keys;
@ -196,8 +198,9 @@ pub async fn get_hexagons(
let need_parent = needs_parent(resolution);
let mut groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
let mut travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> =
(0..travel_entries.len()).map(|_| FxHashMap::default()).collect();
let mut travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> = (0..travel_entries.len())
.map(|_| FxHashMap::default())
.collect();
// Main aggregation loop
let aggregate_row =
@ -246,9 +249,15 @@ pub async fn get_hexagons(
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));
if let Some(sel_indices) = field_indices.as_deref() {
aggregation.add_row_selective(feature_data, row, num_features, sel_indices);
aggregation.add_row_selective(
feature_data,
row,
num_features,
sel_indices,
&quant,
);
} else {
aggregation.add_row(feature_data, row, num_features);
aggregation.add_row(feature_data, row, num_features, &quant);
}
// Aggregate travel time

69
server-rs/src/routes/invites.rs
View file

@ -107,13 +107,23 @@ pub async fn post_invites(
} else if user.subscription == "licensed" {
"referral"
} else {
return (StatusCode::FORBIDDEN, "Only licensed users can create invites").into_response();
return (
StatusCode::FORBIDDEN,
"Only licensed users can create invites",
)
.into_response();
};
let code = generate_invite_code();
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await
let token = match auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await
{
Ok(t) => t,
Err(err) => {
@ -190,7 +200,13 @@ pub async fn get_invite(
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await
let token = match auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await
{
Ok(t) => t,
Err(err) => {
@ -205,9 +221,12 @@ pub async fn get_invite(
urlencoding::encode(&filter)
);
let res = match state.http_client.get(&url)
let res = match state
.http_client
.get(&url)
.header("Authorization", format!("Bearer {token}"))
.send().await
.send()
.await
{
Ok(r) => r,
Err(err) => {
@ -235,8 +254,7 @@ pub async fn get_invite(
// Look up inviter's name (email local part)
let invited_by = if !created_by.is_empty() {
let user_url =
format!("{pb_url}/api/collections/users/records/{created_by}");
let user_url = format!("{pb_url}/api/collections/users/records/{created_by}");
match state
.http_client
.get(&user_url)
@ -245,8 +263,7 @@ pub async fn get_invite(
.await
{
Ok(resp) if resp.status().is_success() => {
let user_body: serde_json::Value =
resp.json().await.unwrap_or_default();
let user_body: serde_json::Value = resp.json().await.unwrap_or_default();
user_body["email"]
.as_str()
.and_then(|e| e.split('@').next())
@ -305,7 +322,13 @@ pub async fn post_redeem_invite(
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await
let token = match auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await
{
Ok(t) => t,
Err(err) => {
@ -315,18 +338,18 @@ pub async fn post_redeem_invite(
};
// Look up invite
let filter = format!(
"code=\"{}\" && used_by_id=\"\"",
req.code
);
let filter = format!("code=\"{}\" && used_by_id=\"\"", req.code);
let lookup_url = format!(
"{pb_url}/api/collections/invites/records?filter={}&perPage=1",
urlencoding::encode(&filter)
);
let res = match state.http_client.get(&lookup_url)
let res = match state
.http_client
.get(&lookup_url)
.header("Authorization", format!("Bearer {token}"))
.send().await
.send()
.await
{
Ok(r) => r,
Err(err) => {
@ -428,7 +451,10 @@ pub async fn post_redeem_invite(
("cancel_url", cancel_url),
("client_reference_id", user.id.clone()),
("customer_email", user.email.clone()),
("discounts[0][coupon]", state.stripe_referral_coupon_id.clone()),
(
"discounts[0][coupon]",
state.stripe_referral_coupon_id.clone(),
),
];
let stripe_res = state
@ -442,10 +468,7 @@ pub async fn post_redeem_invite(
match stripe_res {
Ok(resp) if resp.status().is_success() => {
let stripe_body: serde_json::Value = resp.json().await.unwrap_or_default();
let checkout_url = stripe_body["url"]
.as_str()
.unwrap_or_default()
.to_string();
let checkout_url = stripe_body["url"].as_str().unwrap_or_default().to_string();
info!(user_id = %user.id, code = %req.code, "Referral invite redeemed — checkout created");
Json(RedeemResponse {
result: "checkout".to_string(),
@ -494,9 +517,7 @@ pub async fn get_invites(
format!("created_by=\"{}\"", user.id)
};
let mut url = format!(
"{pb_url}/api/collections/invites/records?sort=-created&perPage=200"
);
let mut url = format!("{pb_url}/api/collections/invites/records?sort=-created&perPage=200");
if !filter.is_empty() {
url.push_str(&format!("&filter={}", urlencoding::encode(&filter)));
}

8
server-rs/src/routes/newsletter.rs
View file

@ -27,7 +27,13 @@ pub async fn patch_newsletter(
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await
let token = match auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await
{
Ok(t) => t,
Err(err) => {

2
server-rs/src/routes/pois.rs
View file

@ -85,7 +85,7 @@ pub async fn get_pois(
let pois: Vec<POI> = matching_rows
.iter()
.map(|&row| POI {
id: state.poi_data.id[row].clone(),
id: state.poi_data.id(row).to_string(),
name: state.poi_data.name[row].clone(),
category: state.poi_data.category.get(row).to_string(),
group: state.poi_data.group.get(row).to_string(),

9
server-rs/src/routes/postcode_properties.rs
View file

@ -46,10 +46,12 @@ pub async fn get_postcode_properties(
check_license_point(&user.0, centroid_lat as f64, centroid_lon as f64)?;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -105,8 +107,11 @@ pub async fn get_postcode_properties(
.take(limit)
.map(|&row| {
super::properties::build_property(
row, &state, feature_names, feature_name_to_index, feature_data,
num_features, enum_values,
row,
&state,
feature_names,
feature_name_to_index,
enum_values,
)
})
.collect();

13
server-rs/src/routes/postcode_stats.rs
View file

@ -50,10 +50,12 @@ pub async fn get_postcode_stats(
// License check using postcode centroid
check_license_point(&user.0, centroid_lat as f64, centroid_lon as f64)?;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -96,18 +98,13 @@ pub async fn get_postcode_stats(
let total_count = matching_rows.len();
let price_history = stats::extract_price_history(
&matching_rows,
feature_data,
num_features,
&state.feature_name_to_index,
);
let price_history =
stats::extract_price_history(&matching_rows, &state.data, &state.feature_name_to_index);
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
&matching_rows,
feature_data,
&state.data,
&state.data.feature_names,
num_features,
&state.data.enum_values,
&state.data.feature_stats,
fields_specified,

15
server-rs/src/routes/postcodes.rs
View file

@ -76,10 +76,12 @@ pub async fn get_postcodes(
check_license_bounds(&user.0, (south, west, north, east))?;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -118,6 +120,7 @@ pub async fn get_postcodes(
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
let quant = state.data.quant_ref();
let min_keys = &state.min_keys;
let max_keys = &state.max_keys;
let avg_keys = &state.avg_keys;
@ -185,18 +188,20 @@ pub async fn get_postcodes(
.or_insert_with(|| Aggregator::new(num_features));
for &row in rows {
if has_selective {
agg.add_row_selective(feature_data, row, num_features, sel_indices);
agg.add_row_selective(feature_data, row, num_features, sel_indices, &quant);
} else {
agg.add_row(feature_data, row, num_features);
agg.add_row(feature_data, row, num_features, &quant);
}
}
// Aggregate travel times for this postcode
if has_travel {
let postcode = &postcode_data.postcodes[pc_idx];
let tt_aggs = travel_aggs
.entry(pc_idx)
.or_insert_with(|| (0..travel_entries.len()).map(|_| TravelTimeAgg::new()).collect());
let tt_aggs = travel_aggs.entry(pc_idx).or_insert_with(|| {
(0..travel_entries.len())
.map(|_| TravelTimeAgg::new())
.collect()
});
for (ti, entry) in travel_entries.iter().enumerate() {
if let Some(row_data) = travel_data[ti].get(postcode.as_str()) {
let minutes = if entry.use_best {

14
server-rs/src/routes/pricing.rs
View file

@ -12,9 +12,9 @@ use crate::state::AppState;
/// Pricing tiers: (cumulative user cap, price in pence).
const TIERS: &[(u64, u64)] = &[
(1, 0), // First 10 users: free
(20, 1000), // Next 10: £10
(45, 2500), // Next 25: £25
(95, 5000), // Next 50: £50
(20, 1000), // Next 10: £10
(45, 2500), // Next 25: £25
(95, 5000), // Next 50: £50
];
const FINAL_PRICE_PENCE: u64 = 10000; // £100 after 95
@ -45,7 +45,13 @@ pub fn price_for_count(count: u64) -> u64 {
/// Count users with subscription="licensed" in PocketBase.
pub async fn count_licensed_users(state: &AppState) -> anyhow::Result<u64> {
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password).await?;
let token = auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await?;
let filter = "subscription=\"licensed\"";
let url = format!(

59
server-rs/src/routes/properties.rs
View file

@ -11,12 +11,12 @@ use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, MAX_PROPERTIES_LIMIT};
use crate::data::RenovationEvent;
use crate::licensing::check_license_bounds;
use crate::parsing::{
cell_for_row, h3_cell_bounds, needs_parent, parse_filters, row_passes_filters,
validate_h3_resolution,
};
use crate::data::RenovationEvent;
use crate::state::AppState;
#[derive(Deserialize)]
@ -78,18 +78,17 @@ fn non_empty_string(text: &str) -> Option<String> {
}
/// Look up an enum feature value by column name.
/// Uses the unified feature model: enum values stored as f32 indices in feature_data.
/// Uses the unified feature model: enum values stored as u16 indices in feature_data.
fn lookup_enum_value(
feature_name_to_index: &FxHashMap<String, usize>,
feature_data: &[f32],
num_features: usize,
data: &crate::data::PropertyData,
enum_values: &FxHashMap<usize, Vec<String>>,
row: usize,
name: &str,
) -> Option<String> {
let &feat_idx = feature_name_to_index.get(name)?;
let values = enum_values.get(&feat_idx)?;
let value = feature_data[row * num_features + feat_idx];
let value = data.get_feature(row, feat_idx);
if value.is_finite() {
let idx = value as usize;
values.get(idx).cloned()
@ -103,17 +102,14 @@ pub fn build_property(
state: &AppState,
feature_names: &[String],
feature_name_to_index: &FxHashMap<String, usize>,
feature_data: &[f32],
num_features: usize,
enum_values: &FxHashMap<usize, Vec<String>>,
) -> Property {
let mut features = FxHashMap::default();
let base = row * num_features;
for (feat_idx, feat_name) in feature_names.iter().enumerate() {
if enum_values.contains_key(&feat_idx) {
continue;
}
let value = feature_data[base + feat_idx];
let value = state.data.get_feature(row, feat_idx);
if value.is_finite() {
features.insert(feat_name.clone(), value);
}
@ -124,26 +120,50 @@ pub fn build_property(
postcode: non_empty_string(state.data.postcode(row)),
is_construction_date_approximate: Some(state.data.is_approx_build_date(row)),
property_type: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type",
feature_name_to_index,
&state.data,
enum_values,
row,
"Property type",
),
built_form: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Property type/built form",
feature_name_to_index,
&state.data,
enum_values,
row,
"Property type/built form",
),
duration: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Leasehold/Freehold",
feature_name_to_index,
&state.data,
enum_values,
row,
"Leasehold/Freehold",
),
current_energy_rating: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Current energy rating",
feature_name_to_index,
&state.data,
enum_values,
row,
"Current energy rating",
),
potential_energy_rating: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Potential energy rating",
feature_name_to_index,
&state.data,
enum_values,
row,
"Potential energy rating",
),
lat: state.data.lat[row],
lon: state.data.lon[row],
renovation_history: state.data.renovation_history(row).to_vec(),
listing_features: state.data.listing_features(row).to_vec(),
listing_status: lookup_enum_value(
feature_name_to_index, feature_data, num_features, enum_values, row, "Listing status",
feature_name_to_index,
&state.data,
enum_values,
row,
"Listing status",
),
listing_url: state.data.listing_url(row).map(String::from),
property_sub_type: state.data.property_sub_type(row).map(String::from),
@ -175,10 +195,12 @@ pub async fn get_hexagon_properties(
check_license_bounds(&user.0, h3_bounds)?;
let h3_str = params.h3;
let quant = state.data.quant_ref();
let (parsed_filters, parsed_enum_filters) = parse_filters(
params.filters.as_deref(),
&state.feature_name_to_index,
&state.data.enum_values,
&quant,
)
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
@ -233,8 +255,11 @@ pub async fn get_hexagon_properties(
.take(limit)
.map(|&row| {
build_property(
row, &state, feature_names, feature_name_to_index, feature_data,
num_features, enum_values,
row,
&state,
feature_names,
feature_name_to_index,
enum_values,
)
})
.collect();

8
server-rs/src/routes/screenshot.rs
View file

@ -7,7 +7,7 @@ use tracing::{info, warn};
use crate::state::AppState;
/// Fetch a PNG screenshot from the screenshot service.
/// Fetch a JPEG screenshot from the screenshot service.
/// Used by both the `/api/screenshot` proxy and the xlsx export.
pub async fn fetch_screenshot_bytes(
state: &AppState,
@ -31,9 +31,7 @@ pub async fn fetch_screenshot_bytes(
Ok(resp) => {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
Err(format!(
"Screenshot service returned {status}: {body}"
))
Err(format!("Screenshot service returned {status}: {body}"))
}
Err(err) => Err(format!("Failed to reach screenshot service: {err}")),
}
@ -51,7 +49,7 @@ pub async fn get_screenshot(
Ok(bytes) => (
StatusCode::OK,
[
(header::CONTENT_TYPE, "image/png"),
(header::CONTENT_TYPE, "image/jpeg"),
(header::CACHE_CONTROL, "public, max-age=86400"),
],
bytes,

9
server-rs/src/routes/shorten.rs
View file

@ -65,9 +65,7 @@ pub async fn post_shorten(state: Arc<AppState>, Json(req): Json<ShortenRequest>)
let res = state
.http_client
.post(format!(
"{pb_url}/api/collections/short_urls/records"
))
.post(format!("{pb_url}/api/collections/short_urls/records"))
.header("Authorization", format!("Bearer {token}"))
.json(&record)
.send()
@ -95,10 +93,7 @@ pub async fn post_shorten(state: Arc<AppState>, Json(req): Json<ShortenRequest>)
}
pub async fn get_short_url(state: Arc<AppState>, Path(code): Path<String>) -> Response {
if code.is_empty()
|| code.len() > 20
|| !code.bytes().all(|b| b.is_ascii_alphanumeric())
{
if code.is_empty() || code.len() > 20 || !code.bytes().all(|b| b.is_ascii_alphanumeric()) {
return StatusCode::BAD_REQUEST.into_response();
}

16
server-rs/src/routes/stats.rs
View file

@ -4,15 +4,14 @@ use rustc_hash::FxHashMap;
use tracing::warn;
use crate::consts::MAX_PRICE_HISTORY_POINTS;
use crate::data::FeatureStats;
use crate::data::{FeatureStats, PropertyData};
use super::hexagon_stats::{EnumFeatureStats, HistogramStats, NumericFeatureStats, PricePoint};
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
pub fn extract_price_history(
matching_rows: &[usize],
feature_data: &[f32],
num_features: usize,
data: &PropertyData,
feature_name_to_index: &FxHashMap<String, usize>,
) -> Vec<PricePoint> {
let year_idx = feature_name_to_index
@ -24,8 +23,8 @@ pub fn extract_price_history(
let mut points: Vec<PricePoint> = matching_rows
.iter()
.filter_map(|&row| {
let year = feature_data[row * num_features + yi];
let price = feature_data[row * num_features + pi];
let year = data.get_feature(row, yi);
let price = data.get_feature(row, pi);
if year.is_finite() && price.is_finite() {
Some(PricePoint { year, price })
} else {
@ -55,9 +54,8 @@ pub fn extract_price_history(
#[allow(clippy::too_many_arguments)]
pub fn compute_feature_stats(
matching_rows: &[usize],
feature_data: &[f32],
data: &PropertyData,
feature_names: &[String],
num_features: usize,
enum_values: &FxHashMap<usize, Vec<String>>,
feature_stats_data: &[FeatureStats],
fields_specified: bool,
@ -74,7 +72,7 @@ pub fn compute_feature_stats(
if let Some(ev) = enum_values.get(&feature_index) {
let mut value_counts = vec![0u64; ev.len()];
for &row in matching_rows {
let value = feature_data[row * num_features + feature_index];
let value = data.get_feature(row, feature_index);
if value.is_finite() {
let idx = value as usize;
if idx < value_counts.len() {
@ -123,7 +121,7 @@ pub fn compute_feature_stats(
};
for &row in matching_rows {
let value = feature_data[row * num_features + feature_index];
let value = data.get_feature(row, feature_index);
if value.is_finite() {
count += 1;
if value < min_value {

24
server-rs/src/routes/stripe_webhook.rs
View file

@ -55,7 +55,10 @@ pub async fn post_stripe_webhook(
) -> Response {
let webhook_secret = &state.stripe_webhook_secret;
let sig_header = match headers.get("stripe-signature").and_then(|h| h.to_str().ok()) {
let sig_header = match headers
.get("stripe-signature")
.and_then(|h| h.to_str().ok())
{
Some(s) => s,
None => {
warn!("Missing Stripe-Signature header");
@ -90,8 +93,13 @@ pub async fn post_stripe_webhook(
// Update user subscription to "licensed" via PocketBase superuser auth
let pb_url = state.pocketbase_url.trim_end_matches('/');
let token = match auth_superuser(&state.http_client, pb_url, &state.pocketbase_admin_email, &state.pocketbase_admin_password)
.await
let token = match auth_superuser(
&state.http_client,
pb_url,
&state.pocketbase_admin_email,
&state.pocketbase_admin_password,
)
.await
{
Ok(t) => t,
Err(err) => {
@ -112,12 +120,18 @@ pub async fn post_stripe_webhook(
match res {
Ok(resp) if resp.status().is_success() => {
state.token_cache.invalidate_by_user_id(user_id);
info!(user_id, "User subscription updated to licensed via Stripe webhook");
info!(
user_id,
"User subscription updated to licensed via Stripe webhook"
);
}
Ok(resp) => {
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
warn!(user_id, "Failed to update user subscription ({status}): {text}");
warn!(
user_id,
"Failed to update user subscription ({status}): {text}"
);
}
Err(err) => {
warn!(user_id, "PocketBase request error in webhook: {err}");

5
server-rs/src/routes/tiles.rs
View file

@ -71,7 +71,10 @@ pub async fn get_style(
.unwrap_or_default();
// Build absolute tile URL using the configured public URL (not the Host header)
let tile_url = format!("{}/api/tiles/{{z}}/{{x}}/{{y}}", public_url.trim_end_matches('/'));
let tile_url = format!(
"{}/api/tiles/{{z}}/{{x}}/{{y}}",
public_url.trim_end_matches('/')
);
let style = build_style(is_dark, &layers, &tile_url);
Ok((

4
server-rs/src/state.rs
View file

@ -3,7 +3,9 @@ use std::sync::Arc;
use rustc_hash::FxHashMap;
use crate::auth::TokenCache;
use crate::data::{POICategoryGroup, POIData, PlaceData, PostcodeData, PropertyData, TravelTimeStore};
use crate::data::{
POICategoryGroup, POIData, PlaceData, PostcodeData, PropertyData, TravelTimeStore,
};
use crate::routes::FeaturesResponse;
use crate::utils::GridIndex;