Optimisations
This commit is contained in:
parent
66c2a25457
commit
9179acd4cd
21 changed files with 653 additions and 139 deletions
10
server-rs/Cargo.lock
generated
10
server-rs/Cargo.lock
generated
|
|
@ -1031,6 +1031,15 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lasso"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb"
|
||||
dependencies = [
|
||||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
|
|
@ -1813,6 +1822,7 @@ dependencies = [
|
|||
"axum",
|
||||
"clap",
|
||||
"h3o",
|
||||
"lasso",
|
||||
"polars",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ h3o = "0.7"
|
|||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
rayon = "1"
|
||||
lasso = "0.7"
|
||||
rustc-hash = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
pub const HISTOGRAM_BINS: usize = 100;
|
||||
|
||||
pub const H3_PRECOMPUTE_MIN: u8 = 4;
|
||||
pub const H3_PRECOMPUTE_MIN: u8 = 7;
|
||||
pub const H3_PRECOMPUTE_MAX: u8 = 12;
|
||||
pub const H3_REQUEST_MIN: u8 = 4;
|
||||
pub const H3_REQUEST_MAX: u8 = 12;
|
||||
|
||||
pub const SERVER_ADDRESS: &str = "0.0.0.0:8001";
|
||||
|
||||
pub const BOUNDS_QUANTIZATION: f64 = 0.01;
|
||||
pub const BOUNDS_BUFFER_PERCENT: f64 = 0.1;
|
||||
pub const GRID_CELL_SIZE: f32 = 0.01;
|
||||
pub const POSTCODE_MIN_RESOLUTION: u8 = 11;
|
||||
pub const MAX_POIS_PER_REQUEST: usize = 2500;
|
||||
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
pub enum Bounds {
|
||||
/// Fixed min/max values for the slider
|
||||
Fixed { min: f64, max: f64 },
|
||||
Fixed { min: f32, max: f32 },
|
||||
/// Compute percentile from data at startup
|
||||
Percentile { low: f64, high: f64 },
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ pub struct FeatureConfig {
|
|||
pub name: &'static str,
|
||||
pub bounds: Bounds,
|
||||
/// Slider step size. Controls the granularity of the range slider in the UI.
|
||||
pub step: f64,
|
||||
pub step: f32,
|
||||
/// Short one-line description shown in the filter sidebar
|
||||
pub description: &'static str,
|
||||
/// Longer description explaining methodology, data source, and caveats
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ use crate::data::EnumFeatureData;
|
|||
|
||||
pub struct ParsedFilter {
|
||||
pub feat_idx: usize,
|
||||
pub min: f64,
|
||||
pub max: f64,
|
||||
pub min: f32,
|
||||
pub max: f32,
|
||||
}
|
||||
|
||||
pub struct ParsedEnumFilter {
|
||||
|
|
@ -51,11 +51,11 @@ pub fn parse_filters(
|
|||
if num_parts.len() != 2 {
|
||||
continue;
|
||||
}
|
||||
let min = match num_parts[0].trim().parse::<f64>() {
|
||||
let min = match num_parts[0].trim().parse::<f32>() {
|
||||
Ok(value) => value,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let max = match num_parts[1].trim().parse::<f64>() {
|
||||
let max = match num_parts[1].trim().parse::<f32>() {
|
||||
Ok(value) => value,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
|
@ -72,7 +72,7 @@ pub fn row_passes_filters(
|
|||
row: usize,
|
||||
filters: &[ParsedFilter],
|
||||
enum_filters: &[ParsedEnumFilter],
|
||||
feature_data: &[f64],
|
||||
feature_data: &[f32],
|
||||
num_features: usize,
|
||||
enum_features: &[EnumFeatureData],
|
||||
) -> bool {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
/// Divides the UK bounding box into cells of ~0.01 degrees (~1km),
|
||||
/// each storing indices of rows whose lat/lon falls within that cell.
|
||||
pub struct GridIndex {
|
||||
min_lat: f64,
|
||||
min_lon: f64,
|
||||
cell_size: f64,
|
||||
min_lat: f32,
|
||||
min_lon: f32,
|
||||
cell_size: f32,
|
||||
cols: usize,
|
||||
rows: usize,
|
||||
/// cells[row * cols + col] = vec of row indices
|
||||
|
|
@ -13,11 +13,11 @@ pub struct GridIndex {
|
|||
}
|
||||
|
||||
impl GridIndex {
|
||||
pub fn build(lat: &[f64], lon: &[f64], cell_size: f64) -> Self {
|
||||
let mut min_lat = f64::INFINITY;
|
||||
let mut max_lat = f64::NEG_INFINITY;
|
||||
let mut min_lon = f64::INFINITY;
|
||||
let mut max_lon = f64::NEG_INFINITY;
|
||||
pub fn build(lat: &[f32], lon: &[f32], cell_size: f32) -> Self {
|
||||
let mut min_lat = f32::INFINITY;
|
||||
let mut max_lat = f32::NEG_INFINITY;
|
||||
let mut min_lon = f32::INFINITY;
|
||||
let mut max_lon = f32::NEG_INFINITY;
|
||||
|
||||
for index in 0..lat.len() {
|
||||
if lat[index] < min_lat {
|
||||
|
|
@ -71,6 +71,7 @@ impl GridIndex {
|
|||
}
|
||||
}
|
||||
|
||||
/// Query accepts f64 bounds (from HTTP parsing) and casts internally.
|
||||
pub fn query(&self, south: f64, west: f64, north: f64, east: f64) -> Vec<u32> {
|
||||
let Some((row_min, row_max, col_min, col_max)) =
|
||||
self.clamp_bounds(south, west, north, east)
|
||||
|
|
@ -121,10 +122,14 @@ impl GridIndex {
|
|||
north: f64,
|
||||
east: f64,
|
||||
) -> Option<(usize, usize, usize, usize)> {
|
||||
let row_min_raw = ((south - self.min_lat) / self.cell_size) as isize;
|
||||
let row_max_raw = ((north - self.min_lat) / self.cell_size) as isize;
|
||||
let col_min_raw = ((west - self.min_lon) / self.cell_size) as isize;
|
||||
let col_max_raw = ((east - self.min_lon) / self.cell_size) as isize;
|
||||
let min_lat = self.min_lat as f64;
|
||||
let min_lon = self.min_lon as f64;
|
||||
let cell_size = self.cell_size as f64;
|
||||
|
||||
let row_min_raw = ((south - min_lat) / cell_size) as isize;
|
||||
let row_max_raw = ((north - min_lat) / cell_size) as isize;
|
||||
let col_min_raw = ((west - min_lon) / cell_size) as isize;
|
||||
let col_max_raw = ((east - min_lon) / cell_size) as isize;
|
||||
|
||||
let row_min = row_min_raw.max(0) as usize;
|
||||
let row_max_clamped = row_max_raw.min(self.rows as isize - 1);
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
);
|
||||
|
||||
info!("Building spatial grid index (0.01° cells)");
|
||||
let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, 0.01);
|
||||
let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, consts::GRID_CELL_SIZE);
|
||||
|
||||
info!(
|
||||
"Precomputing H3 cells for resolutions {}-{}",
|
||||
|
|
@ -89,7 +89,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
info!(pois = poi_data.lat.len(), "POI data loaded");
|
||||
|
||||
info!("Building POI spatial grid index");
|
||||
let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, 0.01);
|
||||
let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);
|
||||
|
||||
let min_keys: Vec<String> = property_data
|
||||
.feature_names
|
||||
|
|
@ -116,11 +116,14 @@ async fn main() -> anyhow::Result<()> {
|
|||
let poi_category_groups = {
|
||||
let mut group_cats: std::collections::HashMap<String, std::collections::HashSet<String>> =
|
||||
std::collections::HashMap::new();
|
||||
for (category, group) in poi_data.category.iter().zip(poi_data.group.iter()) {
|
||||
let num_pois = poi_data.category.indices.len();
|
||||
for row in 0..num_pois {
|
||||
let category = poi_data.category.get(row).to_string();
|
||||
let group = poi_data.group.get(row).to_string();
|
||||
group_cats
|
||||
.entry(group.clone())
|
||||
.entry(group)
|
||||
.or_default()
|
||||
.insert(category.clone());
|
||||
.insert(category);
|
||||
}
|
||||
// Validate that data groups match the hardcoded order exactly
|
||||
let expected: std::collections::HashSet<&str> =
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ pub enum FeatureInfo {
|
|||
#[serde(rename = "numeric")]
|
||||
Numeric {
|
||||
name: String,
|
||||
min: f64,
|
||||
max: f64,
|
||||
step: f64,
|
||||
min: f32,
|
||||
max: f32,
|
||||
step: f32,
|
||||
histogram: Histogram,
|
||||
description: &'static str,
|
||||
detail: &'static str,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use axum::response::IntoResponse;
|
|||
use serde::Deserialize;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{ENUM_NULL, HISTOGRAM_BINS};
|
||||
use crate::consts::{ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, HISTOGRAM_BINS};
|
||||
use crate::filter::{parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
||||
|
|
@ -31,17 +31,21 @@ pub async fn get_hexagon_stats(
|
|||
})?;
|
||||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution as usize;
|
||||
if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() {
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Invalid or non-precomputed resolution for hexagon-stats"
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid or non-precomputed resolution".to_string(),
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
let resolution_idx = resolution as usize;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
|
|
@ -54,7 +58,13 @@ pub async fn get_hexagon_stats(
|
|||
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let start_time = std::time::Instant::now();
|
||||
let h3_data = &state.h3_cells[resolution];
|
||||
let precomputed: Option<&[u64]> = state
|
||||
.h3_cells
|
||||
.get(resolution_idx)
|
||||
.filter(|cells| !cells.is_empty())
|
||||
.map(|cells| cells.as_slice());
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let enum_features = &state.data.enum_features;
|
||||
|
|
@ -67,7 +77,14 @@ pub async fn get_hexagon_stats(
|
|||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if h3_data[row] == cell_u64
|
||||
let row_cell = if let Some(h3_data) = precomputed {
|
||||
h3_data[row]
|
||||
} else {
|
||||
h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if row_cell == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -98,9 +115,9 @@ pub async fn get_hexagon_stats(
|
|||
let bin_width = global_stats.histogram.bin_width;
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f64::INFINITY;
|
||||
let mut max_value = f64::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64; // keep f64 for mean precision
|
||||
let mut bins = vec![0u64; HISTOGRAM_BINS];
|
||||
|
||||
for &row in &matching_rows {
|
||||
|
|
@ -113,12 +130,12 @@ pub async fn get_hexagon_stats(
|
|||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value;
|
||||
sum += value as f64;
|
||||
|
||||
// Bin into histogram using global edges
|
||||
// Bin into histogram using global edges (cast to f64 for bin index math)
|
||||
if bin_width > 0.0 {
|
||||
let bin_index =
|
||||
((value - histogram_min) / bin_width).floor() as isize;
|
||||
((value as f64 - histogram_min as f64) / bin_width as f64).floor() as isize;
|
||||
let clamped_index = bin_index.max(0).min((HISTOGRAM_BINS - 1) as isize) as usize;
|
||||
bins[clamped_index] += 1;
|
||||
}
|
||||
|
|
@ -138,15 +155,15 @@ pub async fn get_hexagon_stats(
|
|||
output.push_str("{\"name\":");
|
||||
write_json_string(&mut output, feature_name);
|
||||
write!(output, ",\"count\":{}", count).unwrap();
|
||||
write!(output, ",\"min\":{}", format_f64(min_value)).unwrap();
|
||||
write!(output, ",\"max\":{}", format_f64(max_value)).unwrap();
|
||||
write!(output, ",\"min\":{}", format_num(min_value)).unwrap();
|
||||
write!(output, ",\"max\":{}", format_num(max_value)).unwrap();
|
||||
write!(output, ",\"mean\":{}", format_f64(mean)).unwrap();
|
||||
output.push_str(",\"histogram\":{\"min\":");
|
||||
write!(output, "{}", format_f64(histogram_min)).unwrap();
|
||||
write!(output, "{}", format_num(histogram_min)).unwrap();
|
||||
output.push_str(",\"max\":");
|
||||
write!(output, "{}", format_f64(histogram_max)).unwrap();
|
||||
write!(output, "{}", format_num(histogram_max)).unwrap();
|
||||
output.push_str(",\"bin_width\":");
|
||||
write!(output, "{}", format_f64(bin_width)).unwrap();
|
||||
write!(output, "{}", format_num(bin_width)).unwrap();
|
||||
output.push_str(",\"counts\":[");
|
||||
for (bin_index, &bin_count) in bins.iter().enumerate() {
|
||||
if bin_index > 0 {
|
||||
|
|
@ -216,10 +233,11 @@ pub async fn get_hexagon_stats(
|
|||
"GET /api/hexagon-stats"
|
||||
);
|
||||
|
||||
output
|
||||
Ok(output)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?;
|
||||
|
||||
Ok((
|
||||
[(axum::http::header::CONTENT_TYPE, "application/json")],
|
||||
|
|
@ -242,6 +260,15 @@ fn write_json_string(output: &mut String, value: &str) {
|
|||
output.push('"');
|
||||
}
|
||||
|
||||
fn format_num(value: f32) -> String {
|
||||
let fv = value as f64;
|
||||
if fv.fract() == 0.0 && fv.abs() < 1e15 {
|
||||
format!("{:.1}", fv)
|
||||
} else {
|
||||
format!("{}", fv)
|
||||
}
|
||||
}
|
||||
|
||||
fn format_f64(value: f64) -> String {
|
||||
if value.fract() == 0.0 && value.abs() < 1e15 {
|
||||
format!("{:.1}", value)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use serde::Deserialize;
|
|||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{
|
||||
BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_PRECOMPUTE_MIN,
|
||||
BOUNDS_BUFFER_PERCENT, BOUNDS_QUANTIZATION, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN,
|
||||
POSTCODE_MIN_RESOLUTION,
|
||||
};
|
||||
use crate::filter::parse_filters;
|
||||
|
|
@ -44,8 +44,8 @@ pub struct HexagonParams {
|
|||
/// Per-cell accumulator for aggregating features
|
||||
struct CellAgg {
|
||||
count: u32,
|
||||
mins: Vec<f64>,
|
||||
maxs: Vec<f64>,
|
||||
mins: Vec<f32>,
|
||||
maxs: Vec<f32>,
|
||||
/// Min/max ordinal indices for enum features (255 = no data yet)
|
||||
enum_mins: Vec<u8>,
|
||||
enum_maxs: Vec<u8>,
|
||||
|
|
@ -60,8 +60,8 @@ impl CellAgg {
|
|||
fn new(num_features: usize, num_enums: usize) -> Self {
|
||||
CellAgg {
|
||||
count: 0,
|
||||
mins: vec![f64::INFINITY; num_features],
|
||||
maxs: vec![f64::NEG_INFINITY; num_features],
|
||||
mins: vec![f32::INFINITY; num_features],
|
||||
maxs: vec![f32::NEG_INFINITY; num_features],
|
||||
enum_mins: vec![ENUM_NULL; num_enums],
|
||||
enum_maxs: vec![0; num_enums],
|
||||
postcode: None,
|
||||
|
|
@ -75,7 +75,7 @@ impl CellAgg {
|
|||
/// feature_data[row * num_features + feat_idx] — all features for one row
|
||||
/// are contiguous, so this reads a single cache line per ~8 features.
|
||||
#[inline]
|
||||
fn add_row(&mut self, feature_data: &[f64], row: usize, num_features: usize) {
|
||||
fn add_row(&mut self, feature_data: &[f32], row: usize, num_features: usize) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
|
|
@ -110,9 +110,9 @@ impl CellAgg {
|
|||
/// Track postcode and centroid for high-resolution cells.
|
||||
/// Uses simple "first seen" approach — at res 11/12, most rows in a cell share a postcode.
|
||||
#[inline]
|
||||
fn add_postcode(&mut self, postcode: &str, lat: f64, lon: f64) {
|
||||
self.lat_sum += lat;
|
||||
self.lon_sum += lon;
|
||||
fn add_postcode(&mut self, postcode: &str, lat: f32, lon: f32) {
|
||||
self.lat_sum += lat as f64;
|
||||
self.lon_sum += lon as f64;
|
||||
if postcode.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
|
@ -212,16 +212,16 @@ pub async fn get_hexagons(
|
|||
Query(params): Query<HexagonParams>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let resolution = params.resolution;
|
||||
if resolution < H3_PRECOMPUTE_MIN || resolution > H3_PRECOMPUTE_MAX {
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_PRECOMPUTE_MIN, H3_PRECOMPUTE_MAX
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
|
|
@ -304,7 +304,7 @@ pub async fn get_hexagons(
|
|||
aggregation.add_enums(enum_features, row);
|
||||
if include_postcode {
|
||||
aggregation.add_postcode(
|
||||
&state.data.postcode[row],
|
||||
state.data.postcode(row),
|
||||
state.data.lat[row],
|
||||
state.data.lon[row],
|
||||
);
|
||||
|
|
@ -320,7 +320,7 @@ pub async fn get_hexagons(
|
|||
if !row_passes(row) {
|
||||
return;
|
||||
}
|
||||
let cell_id = h3o::LatLng::new(state.data.lat[row], state.data.lon[row])
|
||||
let cell_id = h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0);
|
||||
let aggregation = groups
|
||||
|
|
@ -330,7 +330,7 @@ pub async fn get_hexagons(
|
|||
aggregation.add_enums(enum_features, row);
|
||||
if include_postcode {
|
||||
aggregation.add_postcode(
|
||||
&state.data.postcode[row],
|
||||
state.data.postcode(row),
|
||||
state.data.lat[row],
|
||||
state.data.lon[row],
|
||||
);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ pub async fn get_pois(
|
|||
.filter_map(|&row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if let Some(ref categories) = category_filter {
|
||||
if !categories.contains(&state.poi_data.category[row]) {
|
||||
if !categories.contains(state.poi_data.category.get(row)) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
|
@ -83,11 +83,11 @@ pub async fn get_pois(
|
|||
.map(|&row| POI {
|
||||
id: state.poi_data.id[row].clone(),
|
||||
name: state.poi_data.name[row].clone(),
|
||||
category: state.poi_data.category[row].clone(),
|
||||
group: state.poi_data.group[row].clone(),
|
||||
category: state.poi_data.category.get(row).to_string(),
|
||||
group: state.poi_data.group.get(row).to_string(),
|
||||
lat: state.poi_data.lat[row],
|
||||
lng: state.poi_data.lng[row],
|
||||
emoji: state.poi_data.emoji[row].clone(),
|
||||
emoji: state.poi_data.emoji.get(row).to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use rustc_hash::FxHashMap;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, MAX_PROPERTIES_LIMIT};
|
||||
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT};
|
||||
use crate::data::EnumFeatureData;
|
||||
use crate::filter::{parse_filters, row_passes_filters};
|
||||
use crate::state::AppState;
|
||||
|
|
@ -36,13 +36,13 @@ pub struct Property {
|
|||
pub potential_energy_rating: Option<String>,
|
||||
|
||||
// Numeric fields
|
||||
pub lat: f64,
|
||||
pub lon: f64,
|
||||
pub lat: f32,
|
||||
pub lon: f32,
|
||||
|
||||
pub is_construction_date_approximate: Option<bool>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub features: FxHashMap<String, f64>,
|
||||
pub features: FxHashMap<String, f32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -93,17 +93,21 @@ pub async fn get_hexagon_properties(
|
|||
})?;
|
||||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution as usize;
|
||||
if resolution >= state.h3_cells.len() || state.h3_cells[resolution].is_empty() {
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Invalid or non-precomputed resolution for hexagon-properties"
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid or non-precomputed resolution".to_string(),
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
let resolution_idx = resolution as usize;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
|
|
@ -116,7 +120,13 @@ pub async fn get_hexagon_properties(
|
|||
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let t0 = std::time::Instant::now();
|
||||
let h3_data = &state.h3_cells[resolution];
|
||||
let precomputed: Option<&[u64]> = state
|
||||
.h3_cells
|
||||
.get(resolution_idx)
|
||||
.filter(|cells| !cells.is_empty())
|
||||
.map(|cells| cells.as_slice());
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let enum_features = &state.data.enum_features;
|
||||
|
|
@ -128,7 +138,14 @@ pub async fn get_hexagon_properties(
|
|||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if h3_data[row] == cell_u64
|
||||
let row_cell = if let Some(h3_data) = precomputed {
|
||||
h3_data[row]
|
||||
} else {
|
||||
h3o::LatLng::new(state.data.lat[row] as f64, state.data.lon[row] as f64)
|
||||
.map(|coord| u64::from(coord.to_cell(h3_res)))
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if row_cell == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -162,8 +179,8 @@ pub async fn get_hexagon_properties(
|
|||
}
|
||||
|
||||
Property {
|
||||
address: non_empty_string(&state.data.address[row]),
|
||||
postcode: non_empty_string(&state.data.postcode[row]),
|
||||
address: non_empty_string(state.data.address(row)),
|
||||
postcode: non_empty_string(state.data.postcode(row)),
|
||||
is_construction_date_approximate: Some(state.data.is_approx_build_date[row]),
|
||||
property_type: lookup_enum_value(
|
||||
enum_features,
|
||||
|
|
@ -215,16 +232,17 @@ pub async fn get_hexagon_properties(
|
|||
"GET /api/hexagon-properties"
|
||||
);
|
||||
|
||||
HexagonPropertiesResponse {
|
||||
Ok(HexagonPropertiesResponse {
|
||||
properties,
|
||||
total,
|
||||
limit,
|
||||
offset,
|
||||
truncated,
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?;
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
.map_err(|error: String| (StatusCode::INTERNAL_SERVER_ERROR, error))?;
|
||||
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_bounds_fully_below_grid_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(10.0, -10.0, 20.0, -5.0);
|
||||
|
|
@ -17,8 +17,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_bounds_fully_above_grid_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(80.0, 50.0, 90.0, 60.0);
|
||||
|
|
@ -30,8 +30,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_inverted_bounds_returns_empty() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
// south > north
|
||||
|
|
@ -44,8 +44,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn for_each_bounds_fully_outside_yields_nothing() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let mut count = 0;
|
||||
|
|
@ -60,8 +60,8 @@ mod grid_index_tests {
|
|||
fn query_with_large_cells_outside_returns_empty() {
|
||||
// Previously, out-of-bounds queries with large cell sizes would
|
||||
// scan cell (0,0) which could contain data. Now returns empty.
|
||||
let lat = vec![50.0];
|
||||
let lon = vec![0.0];
|
||||
let lat = vec![50.0_f32];
|
||||
let lon = vec![0.0_f32];
|
||||
let grid = GridIndex::build(&lat, &lon, 1.0);
|
||||
|
||||
let results = grid.query(0.0, -50.0, 10.0, -40.0);
|
||||
|
|
@ -73,8 +73,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_within_bounds_returns_correct_results() {
|
||||
let lat = vec![50.0, 50.5, 51.0];
|
||||
let lon = vec![0.0, 0.5, 1.0];
|
||||
let lat = vec![50.0_f32, 50.5, 51.0];
|
||||
let lon = vec![0.0_f32, 0.5, 1.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(49.9, -0.1, 51.1, 1.1);
|
||||
|
|
@ -83,8 +83,8 @@ mod grid_index_tests {
|
|||
|
||||
#[test]
|
||||
fn query_partial_bounds_returns_subset() {
|
||||
let lat = vec![50.0, 51.0, 52.0];
|
||||
let lon = vec![0.0, 0.0, 0.0];
|
||||
let lat = vec![50.0_f32, 51.0, 52.0];
|
||||
let lon = vec![0.0_f32, 0.0, 0.0];
|
||||
let grid = GridIndex::build(&lat, &lon, 0.01);
|
||||
|
||||
let results = grid.query(49.9, -0.1, 50.1, 0.1);
|
||||
|
|
@ -100,7 +100,7 @@ mod filter_tests {
|
|||
#[test]
|
||||
fn nan_rows_fail_numeric_filter_even_with_infinite_range() {
|
||||
let feature_names = vec!["price".to_string()];
|
||||
let feature_data = vec![f64::NAN];
|
||||
let feature_data = vec![f32::NAN];
|
||||
let enum_features: Vec<EnumFeatureData> = vec![];
|
||||
|
||||
let (numeric, enums) =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue