This commit is contained in:
Andras Schmelczer 2026-05-28 21:48:35 +01:00
parent 39ef5c6646
commit c995f12f8b
78 changed files with 4830 additions and 1619 deletions

View file

@ -386,8 +386,7 @@ fn build_school_meta(
let website = extract_optional_str_col(df, "school_website")?.unwrap_or_default();
let telephone = extract_optional_str_col(df, "school_telephone")?.unwrap_or_default();
let head_name = extract_optional_str_col(df, "school_head_name")?.unwrap_or_default();
let ofsted_rating =
extract_optional_str_col(df, "school_ofsted_rating")?.unwrap_or_default();
let ofsted_rating = extract_optional_str_col(df, "school_ofsted_rating")?.unwrap_or_default();
let fetch_str = |col: &Vec<Option<String>>, row: usize| -> Option<String> {
col.get(row).cloned().flatten()

View file

@ -184,7 +184,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
name: "Within conservation area",
order: Some(&["Yes", "No"]),
description: "Whether the postcode point falls inside a designated conservation area",
detail: "Historic England conservation area boundaries, matched to the postcode representative point. The national dataset is indicative rather than definitive, so boundary-sensitive decisions should be checked with the local planning authority.",
detail: "Planning Data conservation area boundaries, matched to the postcode representative point. The national dataset is a work in progress and may include duplicates or incomplete local coverage, so boundary-sensitive decisions should be checked with the local planning authority.",
source: "conservation-areas",
}),
Feature::Enum(EnumFeatureConfig {

View file

@ -167,6 +167,10 @@ struct Cli {
#[arg(long)]
tiles: PathBuf,
/// Optional PMTiles raster basemap for satellite imagery.
#[arg(long, env = "SATELLITE_TILES")]
satellite_tiles: Option<PathBuf>,
/// Optional PMTiles raster overlay for high-resolution strategic noise.
#[arg(long, env = "NOISE_OVERLAY_TILES")]
noise_overlay_tiles: Option<PathBuf>,
@ -475,6 +479,8 @@ async fn main() -> anyhow::Result<()> {
tiles_path,
"noise_lden_10m.pmtiles",
);
let satellite_tiles =
configured_or_default_overlay_path(&cli.satellite_tiles, tiles_path, "satellite.pmtiles");
let crime_hotspot_tiles = configured_or_default_overlay_path(
&cli.crime_hotspot_tiles,
tiles_path,
@ -488,6 +494,7 @@ async fn main() -> anyhow::Result<()> {
let noise_overlay_reader =
init_optional_tile_reader("Noise", noise_overlay_tiles.as_ref()).await?;
let satellite_reader = init_optional_tile_reader("Satellite", satellite_tiles.as_ref()).await?;
let crime_hotspot_reader =
init_optional_tile_reader("Crime hotspots", crime_hotspot_tiles.as_ref()).await?;
let tree_overlay_reader =
@ -692,6 +699,7 @@ async fn main() -> anyhow::Result<()> {
let reader_tile = tile_reader.clone();
let reader_style = tile_reader.clone();
let reader_satellite = satellite_reader.clone();
let reader_noise_overlay = noise_overlay_reader.clone();
let reader_crime_hotspot = crime_hotspot_reader.clone();
let reader_tree_overlay = tree_overlay_reader.clone();
@ -858,6 +866,18 @@ async fn main() -> anyhow::Result<()> {
})
.layer(ConcurrencyLimitLayer::new(20)),
)
.route(
"/api/tiles/satellite/{z}/{x}/{y}",
get(move |path| {
routes::get_overlay_tile(
reader_satellite.clone(),
routes::OverlayTileFormat::RasterJpeg,
"satellite",
path,
)
})
.layer(ConcurrencyLimitLayer::new(30)),
)
.route(
"/api/overlays/noise/{z}/{x}/{y}",
get(move |path| {

View file

@ -6,6 +6,7 @@ mod h3;
pub use bounds::{bounds_intersect, h3_cell_bounds, parse_bounds, require_bounds};
pub use fields::{
parse_enum_dist, parse_field_indices, parse_field_indices_with_poi, parse_field_set,
ParsedFieldIndices,
};
pub use filters::{
count_filter_impacts, count_filter_rejections, parse_filters, parse_filters_with_poi,

View file

@ -40,7 +40,13 @@ pub struct ActualListingsResponse {
pub truncated: bool,
}
const KEEP_UNKNOWN_LISTING_FILTER_FEATURES: &[&str] = &["Total floor area (sqm)"];
const KEEP_UNKNOWN_LISTING_FILTER_FEATURES: &[&str] = &[
"Total floor area (sqm)",
"Leasehold/Freehold",
"Number of bedrooms & living rooms",
"Property type",
];
const LISTING_BOUNDS_EPSILON_DEGREES: f64 = 0.00001;
pub async fn get_actual_listings(
State(shared): State<Arc<SharedState>>,
@ -98,14 +104,29 @@ pub async fn get_actual_listings(
};
let row_indices = actual_listings.grid.query(south, west, north, east);
let total_in_bounds = row_indices.len();
// Build (row, sort_key) pairs so we can sort by index without
// materializing the full ActualListing for every matching row.
let mut matching_rows: Vec<usize> = row_indices
let total_grid_candidates = row_indices.len();
let candidate_rows: Vec<usize> = row_indices
.iter()
.filter_map(|&row_idx| {
let row = row_idx as usize;
row_is_within_bounds(
actual_listings.lat[row],
actual_listings.lon[row],
south,
west,
north,
east,
)
.then_some(row)
})
.collect();
let total_in_bounds = candidate_rows.len();
// Build (row, sort_key) pairs so we can sort by index without
// materializing the full ActualListing for every matching row.
let mut matching_rows: Vec<usize> = candidate_rows
.into_iter()
.filter(|&row| {
if has_listing_filters
&& !row_passes_listing_filters(
row,
@ -116,7 +137,7 @@ pub async fn get_actual_listings(
&keep_unknown_listing_filter_idxs,
)
{
return None;
return false;
}
if has_poi_filters
&& !row_passes_listing_poi_filters(
@ -126,7 +147,7 @@ pub async fn get_actual_listings(
poi_num_features,
)
{
return None;
return false;
}
if has_travel_filters
&& !row_passes_travel_filters(
@ -135,9 +156,9 @@ pub async fn get_actual_listings(
&travel_data,
)
{
return None;
return false;
}
Some(row)
true
})
.collect();
@ -162,6 +183,7 @@ pub async fn get_actual_listings(
results = listings.len(),
total = total_matching,
total_in_bounds,
total_grid_candidates,
offset,
listing_filtered = has_listing_filters,
poi_filtered = has_poi_filters,
@ -214,10 +236,23 @@ fn row_passes_listing_filters(
}
}) && enum_filters.iter().all(|filter| {
let raw = feature_data[base + filter.feat_idx];
raw != NAN_U16 && filter.allowed.contains(&raw)
if raw == NAN_U16 {
keep_unknown_filter_idxs.contains(&filter.feat_idx)
} else {
filter.allowed.contains(&raw)
}
})
}
fn row_is_within_bounds(lat: f32, lon: f32, south: f64, west: f64, north: f64, east: f64) -> bool {
let lat = lat as f64;
let lon = lon as f64;
lat >= south - LISTING_BOUNDS_EPSILON_DEGREES
&& lat <= north + LISTING_BOUNDS_EPSILON_DEGREES
&& lon >= west - LISTING_BOUNDS_EPSILON_DEGREES
&& lon <= east + LISTING_BOUNDS_EPSILON_DEGREES
}
fn row_passes_listing_poi_filters(
row: usize,
filters: &[ParsedPoiFilter],
@ -245,6 +280,20 @@ fn row_passes_listing_poi_filters(
mod tests {
use super::*;
#[test]
fn listing_bounds_check_keeps_only_exact_viewport_rows() {
assert!(row_is_within_bounds(51.5, -0.1, 51.4, -0.2, 51.6, 0.0));
// Bounds are inclusive so edge points are retained.
assert!(row_is_within_bounds(51.4, -0.2, 51.4, -0.2, 51.6, 0.0));
assert!(row_is_within_bounds(51.6, 0.0, 51.4, -0.2, 51.6, 0.0));
assert!(!row_is_within_bounds(51.399, -0.1, 51.4, -0.2, 51.6, 0.0));
assert!(!row_is_within_bounds(51.601, -0.1, 51.4, -0.2, 51.6, 0.0));
assert!(!row_is_within_bounds(51.5, -0.201, 51.4, -0.2, 51.6, 0.0));
assert!(!row_is_within_bounds(51.5, 0.001, 51.4, -0.2, 51.6, 0.0));
}
#[test]
fn listing_floor_area_filter_keeps_unknown_values() {
let floor_area_filter = ParsedFilter {
@ -290,6 +339,48 @@ mod tests {
));
}
#[test]
fn listing_enum_filter_keeps_allowlisted_unknown_values() {
let enum_filter = ParsedEnumFilter {
feat_idx: 0,
allowed: [1u16].into_iter().collect(),
};
let keep_unknown_filter_idxs: FxHashSet<usize> = [0usize].into_iter().collect();
assert!(row_passes_listing_filters(
0,
&[],
&[enum_filter],
&[NAN_U16],
1,
&keep_unknown_filter_idxs
));
assert!(!row_passes_listing_filters(
0,
&[],
&[ParsedEnumFilter {
feat_idx: 0,
allowed: [1u16].into_iter().collect(),
}],
&[2],
1,
&keep_unknown_filter_idxs
));
assert!(row_passes_listing_filters(
0,
&[],
&[ParsedEnumFilter {
feat_idx: 0,
allowed: [1u16].into_iter().collect(),
}],
&[1],
1,
&keep_unknown_filter_idxs
));
}
#[test]
fn listing_poi_filter_uses_listing_metric_matrix() {
let filter = ParsedPoiFilter {

View file

@ -354,7 +354,11 @@ pub async fn get_export(
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
};
let has_poi_filters = !parsed_poi_filters.is_empty();
let filters_str = if is_postcode_mode { None } else { params.filters };
let filters_str = if is_postcode_mode {
None
} else {
params.filters
};
let travel_entries = if is_postcode_mode {
Vec::new()
} else {
@ -472,9 +476,10 @@ pub async fn get_export(
let mut out: Vec<(usize, PostcodeExportAgg)> = Vec::with_capacity(entries.len());
for (pc_idx, _normalized) in entries {
let mut agg = PostcodeExportAgg::new(total_export_features);
for &row_idx in state.data.rows_for_postcode(
&postcode_data.postcodes[*pc_idx],
) {
for &row_idx in state
.data
.rows_for_postcode(&postcode_data.postcodes[*pc_idx])
{
agg.add_row(
feature_data,
row_idx as usize,
@ -518,7 +523,8 @@ pub async fn get_export(
return;
}
if let Some(&pc_idx) = postcode_data.postcode_to_idx.get(postcode) {
by_pc.entry(pc_idx)
by_pc
.entry(pc_idx)
.or_insert_with(|| PostcodeExportAgg::new(total_export_features))
.add_row(
feature_data,
@ -531,10 +537,8 @@ pub async fn get_export(
}
});
let mut aggs: Vec<(usize, PostcodeExportAgg)> = by_pc
.into_iter()
.filter(|(_, agg)| agg.count > 0)
.collect();
let mut aggs: Vec<(usize, PostcodeExportAgg)> =
by_pc.into_iter().filter(|(_, agg)| agg.count > 0).collect();
// Sort by property count descending
aggs.sort_unstable_by_key(|agg| std::cmp::Reverse(agg.1.count));

View file

@ -12,6 +12,7 @@ use super::TileReader;
pub enum OverlayTileFormat {
VectorMvtGzip,
RasterPng,
RasterJpeg,
}
impl OverlayTileFormat {
@ -19,6 +20,7 @@ impl OverlayTileFormat {
match self {
Self::VectorMvtGzip => "application/x-protobuf",
Self::RasterPng => "image/png",
Self::RasterJpeg => "image/jpeg",
}
}

View file

@ -10,7 +10,10 @@ use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::{POSTCODE_SEARCH_OFFSET, PROPERTIES_LIMIT};
use crate::licensing::{check_license_point, resolve_share_code};
use crate::parsing::{parse_filters_with_poi, row_passes_filters, row_passes_poi_filters};
use crate::parsing::{
parse_field_indices_with_poi, parse_filters_with_poi, row_passes_filters,
row_passes_poi_filters,
};
use crate::state::SharedState;
use crate::utils::normalize_postcode;
@ -25,6 +28,10 @@ pub struct PostcodePropertiesParams {
/// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>,
pub offset: Option<usize>,
/// `;;`-separated numeric feature names to include in each property payload.
/// If absent, keeps the legacy behavior and returns all numeric features.
/// If empty, returns only the fixed property card fields.
pub fields: Option<String>,
/// Exact address to rank first when opening properties from address search.
pub focus_address: Option<String>,
/// Share-link code; grants bbox-scoped access for unlicensed users.
@ -76,6 +83,17 @@ pub async fn get_postcode_properties(
let has_poi_filters = !parsed_poi_filters.is_empty();
let travel_entries = parse_optional_travel(params.travel.as_deref())
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let field_indices = parse_field_indices_with_poi(
params.fields.as_deref(),
&state.feature_name_to_index,
&state.data.poi_metrics.name_to_index,
)
.map_err(|err| (err.0, err.1).into_response())?;
let fields_count = field_indices
.normal
.as_ref()
.map(|indices| (indices.len() + field_indices.poi.len()) as i32)
.unwrap_or(-1);
let postcode_str = normalized;
let focus_address = params
@ -165,6 +183,7 @@ pub async fn get_postcode_properties(
feature_names,
feature_name_to_index,
enum_values,
&field_indices,
)
})
.collect();
@ -177,6 +196,7 @@ pub async fn get_postcode_properties(
offset = page_offset,
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
fields = fields_count,
travel_entries = travel_entries.len(),
ms = format_args!("{:.1}", elapsed.as_secs_f64() * 1000.0),
"GET /api/postcode-properties"

View file

@ -14,8 +14,9 @@ use crate::consts::PROPERTIES_LIMIT;
use crate::data::{HistoricalPrice, RenovationEvent};
use crate::licensing::{check_license_bounds, resolve_share_code};
use crate::parsing::{
cell_for_row_cached, h3_cell_bounds, needs_parent, parse_filters_with_poi, row_passes_filters,
row_passes_poi_filters, validate_h3_resolution,
cell_for_row_cached, h3_cell_bounds, needs_parent, parse_field_indices_with_poi,
parse_filters_with_poi, row_passes_filters, row_passes_poi_filters, validate_h3_resolution,
ParsedFieldIndices,
};
use crate::state::{AppState, SharedState};
@ -30,6 +31,10 @@ pub struct HexagonPropertiesParams {
/// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>,
pub offset: Option<usize>,
/// `;;`-separated numeric feature names to include in each property payload.
/// If absent, keeps the legacy behavior and returns all numeric features.
/// If empty, returns only the fixed property card fields.
pub fields: Option<String>,
/// Share-link code; grants bbox-scoped access for unlicensed users.
pub share: Option<String>,
}
@ -106,27 +111,81 @@ fn lookup_enum_value(
}
}
fn insert_feature_value(
features: &mut FxHashMap<String, f32>,
row: usize,
state: &AppState,
feature_names: &[String],
enum_values: &FxHashMap<usize, Vec<String>>,
feat_idx: usize,
) {
if feat_idx >= feature_names.len() || enum_values.contains_key(&feat_idx) {
return;
}
let value = state.data.get_feature(row, feat_idx);
if value.is_finite() {
features.insert(feature_names[feat_idx].clone(), value);
}
}
fn insert_poi_metric_value(
features: &mut FxHashMap<String, f32>,
row: usize,
state: &AppState,
metric_idx: usize,
) {
let Some(metric_name) = state.data.poi_metrics.feature_names.get(metric_idx) else {
return;
};
let value = state.data.poi_metrics.get_for_property_row(row, metric_idx);
if value.is_finite() {
features.insert(metric_name.clone(), value);
}
}
pub fn build_property(
row: usize,
state: &AppState,
feature_names: &[String],
feature_name_to_index: &FxHashMap<String, usize>,
enum_values: &FxHashMap<usize, Vec<String>>,
field_indices: &ParsedFieldIndices,
) -> Property {
let mut features = FxHashMap::default();
for (feat_idx, feat_name) in feature_names.iter().enumerate() {
if enum_values.contains_key(&feat_idx) {
continue;
if let Some(indices) = field_indices.normal.as_deref() {
for &feat_idx in indices {
insert_feature_value(
&mut features,
row,
state,
feature_names,
enum_values,
feat_idx,
);
}
let value = state.data.get_feature(row, feat_idx);
if value.is_finite() {
features.insert(feat_name.clone(), value);
} else {
for feat_idx in 0..feature_names.len() {
insert_feature_value(
&mut features,
row,
state,
feature_names,
enum_values,
feat_idx,
);
}
}
for (metric_idx, metric_name) in state.data.poi_metrics.feature_names.iter().enumerate() {
let value = state.data.poi_metrics.get_for_property_row(row, metric_idx);
if value.is_finite() {
features.insert(metric_name.clone(), value);
if field_indices.normal.is_some() {
for &metric_idx in &field_indices.poi {
insert_poi_metric_value(&mut features, row, state, metric_idx);
}
} else {
for metric_idx in 0..state.data.poi_metrics.feature_names.len() {
insert_poi_metric_value(&mut features, row, state, metric_idx);
}
}
@ -241,6 +300,17 @@ pub async fn get_hexagon_properties(
let has_poi_filters = !parsed_poi_filters.is_empty();
let travel_entries = parse_optional_travel(params.travel.as_deref())
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?;
let field_indices = parse_field_indices_with_poi(
params.fields.as_deref(),
&state.feature_name_to_index,
&state.data.poi_metrics.name_to_index,
)
.map_err(|err| (err.0, err.1).into_response())?;
let fields_count = field_indices
.normal
.as_ref()
.map(|indices| (indices.len() + field_indices.poi.len()) as i32)
.unwrap_or(-1);
let result = tokio::task::spawn_blocking(move || {
let t0 = std::time::Instant::now();
@ -309,6 +379,7 @@ pub async fn get_hexagon_properties(
feature_names,
feature_name_to_index,
enum_values,
&field_indices,
)
})
.collect();
@ -322,6 +393,7 @@ pub async fn get_hexagon_properties(
offset,
filters = num_filters,
filters_raw = filters_str.as_deref().unwrap_or("-"),
fields = fields_count,
travel_entries = travel_entries.len(),
ms = format_args!("{:.1}", elapsed.as_secs_f64() * 1000.0),
"GET /api/hexagon-properties"

View file

@ -135,6 +135,7 @@ fn is_allowed_param_key(key: &str) -> bool {
| "amenityCount5km"
| "poi"
| "overlay"
| "basemap"
| "tab"
| "pc"
| "tt"
@ -585,6 +586,14 @@ mod tests {
);
}
#[test]
fn preserves_basemap_for_share_links() {
let params =
sanitized_query_params("lat=51.5&lon=-0.1&zoom=12&basemap=satellite", false).unwrap();
assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
}
#[test]
fn escapes_html_attributes() {
assert_eq!(escape_attr(r#""'><&"#), "&quot;&#39;&gt;&lt;&amp;");