andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Refactor and other improvements

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-08 18:25:58 +00:00
parent 04a78e7bfe
commit 6c90cf3c0f
47 changed files with 2705 additions and 1568 deletions
Download patch file Download diff file Expand all files Collapse all files

179
server-rs/src/routes/postcode_stats.rs
View file

@ -1,4 +1,3 @@
use std::collections::HashMap;
use std::sync::Arc;
use axum::extract::Query;
@ -7,12 +6,12 @@ use axum::response::Json;
use serde::Deserialize;
use tracing::{info, warn};
use crate::parsing::{parse_filters, row_passes_filters};
use crate::consts::POSTCODE_SEARCH_OFFSET;
use crate::parsing::{parse_field_set, parse_filters, row_passes_filters};
use crate::state::AppState;
use super::hexagon_stats::{
EnumFeatureStats, HexagonStatsResponse, HistogramStats, NumericFeatureStats, PricePoint,
};
use super::hexagon_stats::HexagonStatsResponse;
use super::stats;
#[derive(Deserialize)]
pub struct PostcodeStatsParams {
@ -56,18 +55,7 @@ pub async fn get_postcode_stats(
);
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
let fields_specified = params.fields.is_some();
let field_set: std::collections::HashSet<String> = params
.fields
.as_ref()
.map(|fields_str| {
fields_str
.split(',')
.map(|field| field.trim().to_string())
.filter(|field| !field.is_empty())
.collect()
})
.unwrap_or_default();
let (fields_specified, field_set) = parse_field_set(params.fields.as_deref());
let postcode_str = normalized.clone();
@ -76,8 +64,8 @@ pub async fn get_postcode_stats(
let num_features = state.data.num_features;
let feature_data = &state.data.feature_data;
// Search ±0.02° around centroid (~2km, generous for a postcode)
let offset: f64 = 0.02;
// Search around centroid (generous for a postcode)
let offset: f64 = POSTCODE_SEARCH_OFFSET;
let min_lat = centroid_lat as f64 - offset;
let max_lat = centroid_lat as f64 + offset;
let min_lon = centroid_lon as f64 - offset;
@ -104,144 +92,23 @@ pub async fn get_postcode_stats(
let total_count = matching_rows.len();
// Collect price history (year, price) pairs
let price_history = {
let year_idx = state
.feature_name_to_index
.get("Date of last transaction")
.copied();
let price_idx = state.feature_name_to_index.get("Last known price").copied();
match (year_idx, price_idx) {
(Some(yi), Some(pi)) => {
let mut points: Vec<PricePoint> = matching_rows
.iter()
.filter_map(|&row| {
let year = feature_data[row * num_features + yi];
let price = feature_data[row * num_features + pi];
if year.is_finite() && price.is_finite() {
Some(PricePoint { year, price })
} else {
None
}
})
.collect();
// Cap at 5000 points by evenly sampling
if points.len() > 5000 {
let step = points.len() as f64 / 5000.0;
points = (0..5000)
.map(|i| {
let idx = (i as f64 * step) as usize;
PricePoint {
year: points[idx].year,
price: points[idx].price,
}
})
.collect();
}
points
}
_ => Vec::new(),
}
};
let price_history = stats::extract_price_history(
&matching_rows,
feature_data,
num_features,
&state.feature_name_to_index,
);
let mut numeric_features = Vec::new();
let mut enum_features_out = Vec::new();
for (feature_index, feature_name) in state.data.feature_names.iter().enumerate() {
if fields_specified && !field_set.contains(feature_name.as_str()) {
continue;
}
if let Some(enum_values) = state.data.enum_values.get(&feature_index) {
// Enum feature: count occurrences of each value
let mut value_counts = vec![0u64; enum_values.len()];
for &row in &matching_rows {
let value = feature_data[row * num_features + feature_index];
if value.is_finite() {
let idx = value as usize;
if idx < value_counts.len() {
value_counts[idx] += 1;
}
}
}
let counts: HashMap<String, u64> = value_counts
.iter()
.enumerate()
.filter(|(_, &count)| count > 0)
.map(|(idx, &count)| (enum_values[idx].clone(), count))
.collect();
if !counts.is_empty() {
enum_features_out.push(EnumFeatureStats {
name: feature_name.clone(),
counts,
});
}
} else {
// Numeric feature: compute stats and histogram
let global_hist = &state.data.feature_stats[feature_index].histogram;
let p1 = global_hist.p1;
let p99 = global_hist.p99;
let num_bins = global_hist.counts.len();
let mut count = 0usize;
let mut min_value = f32::INFINITY;
let mut max_value = f32::NEG_INFINITY;
let mut sum = 0.0f64;
let mut bins = vec![0u64; num_bins];
let middle_bins = num_bins.saturating_sub(2);
let middle_width = if middle_bins > 0 && p99 > p1 {
(p99 - p1) / middle_bins as f32
} else {
0.0
};
for &row in &matching_rows {
let value = feature_data[row * num_features + feature_index];
if value.is_finite() {
count += 1;
if value < min_value {
min_value = value;
}
if value > max_value {
max_value = value;
}
sum += value as f64;
let bin = if value < p1 {
0
} else if value >= p99 {
num_bins - 1
} else if middle_width > 0.0 {
let middle_bin = ((value - p1) / middle_width) as usize;
(1 + middle_bin).min(num_bins - 2)
} else {
num_bins / 2
};
bins[bin] += 1;
}
}
if count > 0 {
numeric_features.push(NumericFeatureStats {
name: feature_name.clone(),
count,
min: min_value as f64,
max: max_value as f64,
mean: sum / count as f64,
histogram: HistogramStats {
min: global_hist.min as f64,
max: global_hist.max as f64,
p1: p1 as f64,
p99: p99 as f64,
counts: bins,
},
});
}
}
}
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
&matching_rows,
feature_data,
&state.data.feature_names,
num_features,
&state.data.enum_values,
&state.data.feature_stats,
fields_specified,
&field_set,
);
let elapsed = start_time.elapsed();
info!(