lgtm
This commit is contained in:
parent
f7e0814a38
commit
fd2860070a
55 changed files with 4084 additions and 186 deletions
|
|
@ -1,7 +1,10 @@
|
|||
mod actual_listings;
|
||||
pub mod area_crime_averages;
|
||||
pub mod crime_by_year;
|
||||
mod developments;
|
||||
mod places;
|
||||
mod poi;
|
||||
pub mod postcode_population;
|
||||
mod postcodes;
|
||||
mod property;
|
||||
pub mod spill;
|
||||
|
|
@ -60,11 +63,14 @@ where
|
|||
}
|
||||
|
||||
pub use actual_listings::{ActualListing, ActualListingData};
|
||||
pub use area_crime_averages::AreaCrimeAverages;
|
||||
pub use crime_by_year::CrimeByYearData;
|
||||
pub use developments::{DevelopmentData, DevelopmentSite};
|
||||
pub use places::{
|
||||
compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,
|
||||
};
|
||||
pub use poi::{resolve_poi_category_filter, POICategoryGroup, POIData, SchoolMetadata};
|
||||
pub use postcode_population::PostcodePopulation;
|
||||
pub use postcodes::{OutcodeData, PostcodeData};
|
||||
pub use property::{
|
||||
precompute_h3, FeatureStats, Histogram, HistoricalPrice, PostcodePoiMetrics, PropertyData,
|
||||
|
|
|
|||
47
server-rs/src/data/area_crime_averages.rs
Normal file
47
server-rs/src/data/area_crime_averages.rs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
//! Precomputed per-outcode and per-postcode-sector average crime rates.
|
||||
//!
|
||||
//! The right pane shows each crime metric's national average (the global
|
||||
//! feature-histogram mean). To let users see how an area compares with its
|
||||
//! immediate surroundings, we also precompute the mean headline crime rate
|
||||
//! (`"X (avg/yr)"`) across every property in the selection's outcode (e.g.
|
||||
//! `"E14"`) and postcode sector (e.g. `"E14 2"`).
|
||||
//!
|
||||
//! Crime figures are constant within a postcode (the pipeline merges them on
|
||||
//! the postcode key), so each postcode's value is read once — from its first
|
||||
//! row — and property-weighted by the postcode's row count. That keeps these
|
||||
//! averages on the same property-weighted basis as the national average, so the
|
||||
//! four numbers (this area / sector / outcode / nation) are directly comparable.
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
/// Crime-feature name suffix that marks an annualised headline-rate column
|
||||
/// (e.g. `"Burglary (avg/yr)"`). Stripped to derive the bare type name.
|
||||
pub const AVG_YR_SUFFIX: &str = " (avg/yr)";
|
||||
|
||||
pub struct AreaCrimeAverages {
|
||||
/// Bare crime-type names (suffix stripped, e.g. `"Burglary"`), index-aligned
|
||||
/// with the per-area mean vectors. Matches `CrimeYearStats.name`.
|
||||
pub crime_types: Vec<String>,
|
||||
/// National mean headline rate per crime type (index-aligned with
|
||||
/// `crime_types`). An EXACT property-weighted mean over every postcode, so it
|
||||
/// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
|
||||
/// unlike the histogram-bin national average, which is biased upward for the
|
||||
/// right-skewed crime densities. `NaN` where no postcode has data.
|
||||
pub national: Vec<f32>,
|
||||
/// Outcode (e.g. `"E14"`) → mean headline rate per crime type. `NaN` where
|
||||
/// the outcode has no data for that type.
|
||||
pub by_outcode: FxHashMap<String, Vec<f32>>,
|
||||
/// Postcode sector (e.g. `"E14 2"`) → mean headline rate per crime type.
|
||||
pub by_sector: FxHashMap<String, Vec<f32>>,
|
||||
}
|
||||
|
||||
impl AreaCrimeAverages {
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
crime_types: Vec::new(),
|
||||
national: Vec::new(),
|
||||
by_outcode: FxHashMap::default(),
|
||||
by_sector: FxHashMap::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
343
server-rs/src/data/developments.rs
Normal file
343
server-rs/src/data/developments.rs
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use polars::lazy::frame::LazyFrame;
|
||||
use polars::prelude::*;
|
||||
use serde::Serialize;
|
||||
use tracing::info;
|
||||
|
||||
use crate::utils::GridIndex;
|
||||
|
||||
const GRID_CELL_SIZE: f32 = 0.01;
|
||||
|
||||
/// A single planned/pipeline development site (one brownfield-register entry or
|
||||
/// one Homes England land-disposal site). These are *sites*, not properties:
|
||||
/// they carry a coordinate, an estimate of the number of new dwellings, and the
|
||||
/// planning status — the forward-looking "where new homes are coming" signal.
|
||||
#[derive(Serialize, Clone)]
|
||||
pub struct DevelopmentSite {
|
||||
pub lat: f32,
|
||||
pub lon: f32,
|
||||
/// Data source: "brownfield" (MHCLG Brownfield Land register) or
|
||||
/// "homes-england" (Homes England Land Hub).
|
||||
pub source: String,
|
||||
pub name: Option<String>,
|
||||
pub min_dwellings: Option<i32>,
|
||||
pub max_dwellings: Option<i32>,
|
||||
pub planning_status: Option<String>,
|
||||
pub permission_type: Option<String>,
|
||||
pub permission_date: Option<String>,
|
||||
pub hectares: Option<f32>,
|
||||
pub local_authority: Option<String>,
|
||||
pub url: Option<String>,
|
||||
}
|
||||
|
||||
/// Columnar in-memory store of development sites with a spatial grid index for
|
||||
/// viewport queries. Small enough (~40k rows nationally) to keep on the heap; no
|
||||
/// quantization or spill needed.
|
||||
pub struct DevelopmentData {
|
||||
pub lat: Vec<f32>,
|
||||
pub lon: Vec<f32>,
|
||||
source: Vec<String>,
|
||||
name: Vec<Option<String>>,
|
||||
min_dwellings: Vec<Option<i32>>,
|
||||
max_dwellings: Vec<Option<i32>>,
|
||||
planning_status: Vec<Option<String>>,
|
||||
permission_type: Vec<Option<String>>,
|
||||
permission_date: Vec<Option<String>>,
|
||||
hectares: Vec<Option<f32>>,
|
||||
local_authority: Vec<Option<String>>,
|
||||
url: Vec<Option<String>>,
|
||||
grid: GridIndex,
|
||||
}
|
||||
|
||||
impl DevelopmentData {
|
||||
/// Empty store, used only by tests (the `--developments` parquet is required
|
||||
/// in production).
|
||||
#[cfg(test)]
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
lat: Vec::new(),
|
||||
lon: Vec::new(),
|
||||
source: Vec::new(),
|
||||
name: Vec::new(),
|
||||
min_dwellings: Vec::new(),
|
||||
max_dwellings: Vec::new(),
|
||||
planning_status: Vec::new(),
|
||||
permission_type: Vec::new(),
|
||||
permission_date: Vec::new(),
|
||||
hectares: Vec::new(),
|
||||
local_authority: Vec::new(),
|
||||
url: Vec::new(),
|
||||
grid: GridIndex::build(&[], &[], GRID_CELL_SIZE),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(parquet_path: &Path) -> Result<Self> {
|
||||
super::run_polars_io(|| Self::load_inner(parquet_path))
|
||||
}
|
||||
|
||||
fn load_inner(parquet_path: &Path) -> Result<Self> {
|
||||
info!("Loading development sites from {:?}", parquet_path);
|
||||
let pl_path = PlRefPath::try_from_path(parquet_path)
|
||||
.context("Failed to normalize development sites parquet path")?;
|
||||
let df = LazyFrame::scan_parquet(pl_path, Default::default())
|
||||
.context("Failed to scan development sites parquet")?
|
||||
.collect()
|
||||
.context("Failed to read development sites parquet")?;
|
||||
|
||||
let lat = extract_f32(&df, "lat")?;
|
||||
let lon = extract_f32(&df, "lon")?;
|
||||
let source = extract_str(&df, "source")?;
|
||||
let name = extract_opt_str(&df, "name")?;
|
||||
let min_dwellings = extract_opt_i32(&df, "min_dwellings")?;
|
||||
let max_dwellings = extract_opt_i32(&df, "max_dwellings")?;
|
||||
let planning_status = extract_opt_str(&df, "planning_status")?;
|
||||
let permission_type = extract_opt_str(&df, "permission_type")?;
|
||||
let permission_date = extract_opt_str(&df, "permission_date")?;
|
||||
let hectares = extract_opt_f32(&df, "hectares")?;
|
||||
let local_authority = extract_opt_str(&df, "local_authority")?;
|
||||
let url = extract_opt_str(&df, "url")?;
|
||||
|
||||
let grid = GridIndex::build(&lat, &lon, GRID_CELL_SIZE);
|
||||
|
||||
info!(rows = lat.len(), "Development sites loaded");
|
||||
|
||||
Ok(Self {
|
||||
lat,
|
||||
lon,
|
||||
source,
|
||||
name,
|
||||
min_dwellings,
|
||||
max_dwellings,
|
||||
planning_status,
|
||||
permission_type,
|
||||
permission_date,
|
||||
hectares,
|
||||
local_authority,
|
||||
url,
|
||||
grid,
|
||||
})
|
||||
}
|
||||
|
||||
fn site_at(&self, row: usize) -> DevelopmentSite {
|
||||
DevelopmentSite {
|
||||
lat: self.lat[row],
|
||||
lon: self.lon[row],
|
||||
source: self.source[row].clone(),
|
||||
name: self.name[row].clone(),
|
||||
min_dwellings: self.min_dwellings[row],
|
||||
max_dwellings: self.max_dwellings[row],
|
||||
planning_status: self.planning_status[row].clone(),
|
||||
permission_type: self.permission_type[row].clone(),
|
||||
permission_date: self.permission_date[row].clone(),
|
||||
hectares: self.hectares[row],
|
||||
local_authority: self.local_authority[row].clone(),
|
||||
url: self.url[row].clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return every site whose coordinate falls within the bounds, sorted by the
|
||||
/// largest dwelling estimate first (so the biggest schemes survive the cap),
|
||||
/// capped at `limit`. The bool is true when the result was truncated.
|
||||
pub fn query_bounds(
|
||||
&self,
|
||||
south: f64,
|
||||
west: f64,
|
||||
north: f64,
|
||||
east: f64,
|
||||
limit: usize,
|
||||
) -> (Vec<DevelopmentSite>, usize, bool) {
|
||||
let mut rows: Vec<usize> = self
|
||||
.grid
|
||||
.query(south, west, north, east)
|
||||
.into_iter()
|
||||
.filter_map(|row_idx| {
|
||||
let row = row_idx as usize;
|
||||
row_within_bounds(self.lat[row], self.lon[row], south, west, north, east)
|
||||
.then_some(row)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let total = rows.len();
|
||||
|
||||
// Biggest schemes first: rank on the upper dwelling estimate, falling back
|
||||
// to the lower estimate, so a viewport that exceeds the cap still surfaces
|
||||
// the most significant developments.
|
||||
rows.sort_by(|&a, &b| {
|
||||
let key = |row: usize| {
|
||||
self.max_dwellings[row]
|
||||
.or(self.min_dwellings[row])
|
||||
.unwrap_or(0)
|
||||
};
|
||||
key(b).cmp(&key(a))
|
||||
});
|
||||
|
||||
let truncated = total > limit;
|
||||
let sites = rows
|
||||
.into_iter()
|
||||
.take(limit)
|
||||
.map(|row| self.site_at(row))
|
||||
.collect();
|
||||
(sites, total, truncated)
|
||||
}
|
||||
}
|
||||
|
||||
fn row_within_bounds(lat: f32, lon: f32, south: f64, west: f64, north: f64, east: f64) -> bool {
|
||||
let lat = lat as f64;
|
||||
let lon = lon as f64;
|
||||
lat >= south && lat <= north && lon >= west && lon <= east
|
||||
}
|
||||
|
||||
fn extract_f32(df: &DataFrame, name: &str) -> Result<Vec<f32>> {
|
||||
let cast = df
|
||||
.column(name)
|
||||
.with_context(|| format!("Missing column '{name}'"))?
|
||||
.cast(&DataType::Float32)
|
||||
.with_context(|| format!("Failed to cast '{name}' to Float32"))?;
|
||||
let column = cast
|
||||
.f32()
|
||||
.with_context(|| format!("Column '{name}' is not Float32"))?;
|
||||
column
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(row, value)| value.with_context(|| format!("Column '{name}' has null at row {row}")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn extract_str(df: &DataFrame, name: &str) -> Result<Vec<String>> {
|
||||
let column = df
|
||||
.column(name)
|
||||
.with_context(|| format!("Missing column '{name}'"))?;
|
||||
let strings = column
|
||||
.str()
|
||||
.with_context(|| format!("Column '{name}' is not a string column"))?;
|
||||
strings
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(row, value)| {
|
||||
value
|
||||
.map(ToString::to_string)
|
||||
.with_context(|| format!("Column '{name}' has null at row {row}"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn extract_opt_str(df: &DataFrame, name: &str) -> Result<Vec<Option<String>>> {
|
||||
let column = df
|
||||
.column(name)
|
||||
.with_context(|| format!("Missing column '{name}'"))?;
|
||||
let strings = column
|
||||
.str()
|
||||
.with_context(|| format!("Column '{name}' is not a string column"))?;
|
||||
Ok(strings
|
||||
.into_iter()
|
||||
.map(|value| value.and_then(|text| (!text.trim().is_empty()).then(|| text.to_string())))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn extract_opt_i32(df: &DataFrame, name: &str) -> Result<Vec<Option<i32>>> {
|
||||
let cast = df
|
||||
.column(name)
|
||||
.with_context(|| format!("Missing column '{name}'"))?
|
||||
.cast(&DataType::Int32)
|
||||
.with_context(|| format!("Failed to cast '{name}' to Int32"))?;
|
||||
let column = cast
|
||||
.i32()
|
||||
.with_context(|| format!("Column '{name}' is not Int32"))?;
|
||||
Ok(column.into_iter().collect())
|
||||
}
|
||||
|
||||
fn extract_opt_f32(df: &DataFrame, name: &str) -> Result<Vec<Option<f32>>> {
|
||||
let cast = df
|
||||
.column(name)
|
||||
.with_context(|| format!("Missing column '{name}'"))?
|
||||
.cast(&DataType::Float32)
|
||||
.with_context(|| format!("Failed to cast '{name}' to Float32"))?;
|
||||
let column = cast
|
||||
.f32()
|
||||
.with_context(|| format!("Column '{name}' is not Float32"))?;
|
||||
Ok(column
|
||||
.into_iter()
|
||||
.map(|value| value.filter(|v| v.is_finite()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn build(points: &[(f32, f32, Option<i32>)]) -> DevelopmentData {
|
||||
let lat: Vec<f32> = points.iter().map(|p| p.0).collect();
|
||||
let lon: Vec<f32> = points.iter().map(|p| p.1).collect();
|
||||
let grid = GridIndex::build(&lat, &lon, GRID_CELL_SIZE);
|
||||
DevelopmentData {
|
||||
source: points.iter().map(|_| "brownfield".to_string()).collect(),
|
||||
name: points.iter().map(|_| None).collect(),
|
||||
min_dwellings: points.iter().map(|_| None).collect(),
|
||||
max_dwellings: points.iter().map(|p| p.2).collect(),
|
||||
planning_status: points.iter().map(|_| None).collect(),
|
||||
permission_type: points.iter().map(|_| None).collect(),
|
||||
permission_date: points.iter().map(|_| None).collect(),
|
||||
hectares: points.iter().map(|_| None).collect(),
|
||||
local_authority: points.iter().map(|_| None).collect(),
|
||||
url: points.iter().map(|_| None).collect(),
|
||||
lat,
|
||||
lon,
|
||||
grid,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_returns_only_in_bounds_sites() {
|
||||
let data = build(&[
|
||||
(51.50, -0.10, Some(5)), // inside
|
||||
(51.55, -0.05, Some(50)), // inside
|
||||
(52.50, -1.00, Some(99)), // outside
|
||||
]);
|
||||
let (sites, total, truncated) = data.query_bounds(51.4, -0.2, 51.6, 0.0, 100);
|
||||
assert_eq!(total, 2);
|
||||
assert!(!truncated);
|
||||
// Sorted by max_dwellings desc: the 50-dwelling scheme comes first.
|
||||
assert_eq!(sites[0].max_dwellings, Some(50));
|
||||
assert_eq!(sites[1].max_dwellings, Some(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_caps_and_flags_truncation_keeping_biggest() {
|
||||
let data = build(&[
|
||||
(51.50, -0.10, Some(1)),
|
||||
(51.51, -0.11, Some(900)),
|
||||
(51.52, -0.12, Some(10)),
|
||||
]);
|
||||
let (sites, total, truncated) = data.query_bounds(51.4, -0.2, 51.6, 0.0, 1);
|
||||
assert_eq!(total, 3);
|
||||
assert!(truncated);
|
||||
assert_eq!(sites.len(), 1);
|
||||
assert_eq!(sites[0].max_dwellings, Some(900));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_store_is_empty() {
|
||||
let data = DevelopmentData::empty();
|
||||
assert!(data.lat.is_empty());
|
||||
let (sites, total, truncated) = data.query_bounds(51.4, -0.2, 51.6, 0.0, 100);
|
||||
assert!(sites.is_empty());
|
||||
assert_eq!(total, 0);
|
||||
assert!(!truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loads_sample_parquet_when_available() {
|
||||
let path = PathBuf::from("../property-data/development_sites.parquet");
|
||||
if !path.exists() {
|
||||
eprintln!("sample development_sites.parquet not present; skipping");
|
||||
return;
|
||||
}
|
||||
let data = DevelopmentData::load_inner(&path).expect("developments load");
|
||||
assert!(!data.lat.is_empty());
|
||||
assert_eq!(data.lat.len(), data.lon.len());
|
||||
assert_eq!(data.lat.len(), data.source.len());
|
||||
}
|
||||
}
|
||||
125
server-rs/src/data/postcode_population.rs
Normal file
125
server-rs/src/data/postcode_population.rs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
//! Per-unit-postcode usual-resident headcounts (ONS Census 2021, table P001),
|
||||
//! loaded from a side parquet and shown in the right pane. This is display-only
|
||||
//! area data: it is never a filterable attribute and never enters the feature
|
||||
//! matrix, mirroring the crime-by-year side table.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{bail, Context};
|
||||
use polars::prelude::PlRefPath;
|
||||
use polars::prelude::*;
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::info;
|
||||
|
||||
use crate::utils::normalize_postcode;
|
||||
|
||||
use super::run_polars_io;
|
||||
|
||||
pub struct PostcodePopulation {
|
||||
/// Canonical spaced postcode (e.g. "AL1 1AG") → usual residents (Census 2021).
|
||||
by_postcode: FxHashMap<String, u32>,
|
||||
}
|
||||
|
||||
impl PostcodePopulation {
|
||||
/// Empty table — used in tests and when no --population-path is supplied.
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
by_postcode: FxHashMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||
run_polars_io(|| Self::load_inner(path))
|
||||
}
|
||||
|
||||
fn load_inner(path: &Path) -> anyhow::Result<Self> {
|
||||
info!("Loading postcode population from {}", path.display());
|
||||
let pl_path = PlRefPath::try_from_path(path).with_context(|| {
|
||||
format!(
|
||||
"Failed to normalize population parquet path {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
let df = LazyFrame::scan_parquet(pl_path, Default::default())
|
||||
.with_context(|| format!("Failed to scan population parquet at {}", path.display()))?
|
||||
.collect()
|
||||
.with_context(|| format!("Failed to read population parquet at {}", path.display()))?;
|
||||
|
||||
let postcode_col = df
|
||||
.column("postcode")
|
||||
.context("population parquet missing 'postcode' column")?
|
||||
.str()
|
||||
.context("'postcode' column is not a string")?;
|
||||
// Accept whatever integer width the parquet writer used.
|
||||
let population_cast = df
|
||||
.column("population")
|
||||
.context("population parquet missing 'population' column")?
|
||||
.cast(&DataType::Int64)
|
||||
.context("'population' column is not an integer")?;
|
||||
let population_col = population_cast
|
||||
.i64()
|
||||
.context("failed to read 'population' as i64")?;
|
||||
|
||||
let mut by_postcode: FxHashMap<String, u32> = FxHashMap::default();
|
||||
by_postcode.reserve(df.height());
|
||||
for (postcode, population) in postcode_col.into_iter().zip(population_col.into_iter()) {
|
||||
let (Some(postcode), Some(population)) = (postcode, population) else {
|
||||
continue;
|
||||
};
|
||||
let trimmed = postcode.trim();
|
||||
if trimmed.is_empty() || population <= 0 {
|
||||
continue;
|
||||
}
|
||||
// Normalize to the exact canonical form the routes look up with, so
|
||||
// a stray double-space or lowercase in the source can't miss.
|
||||
by_postcode.insert(
|
||||
normalize_postcode(trimmed),
|
||||
population.min(u32::MAX as i64) as u32,
|
||||
);
|
||||
}
|
||||
|
||||
if by_postcode.is_empty() {
|
||||
bail!("population parquet at {} produced no rows", path.display());
|
||||
}
|
||||
|
||||
info!(
|
||||
postcodes = by_postcode.len(),
|
||||
"Postcode population loaded"
|
||||
);
|
||||
Ok(Self { by_postcode })
|
||||
}
|
||||
|
||||
/// Usual-resident count for a single canonical (spaced, upper-case) postcode.
|
||||
pub fn for_postcode(&self, postcode: &str) -> Option<u32> {
|
||||
self.by_postcode.get(postcode).copied()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Integration smoke test against the real Census 2021 parquet. Skips when
|
||||
/// the data file is absent (CI without a data build) so it never blocks.
|
||||
#[test]
|
||||
fn loads_real_census_parquet_if_present() {
|
||||
let path = std::path::Path::new("../property-data/population_by_postcode.parquet");
|
||||
if !path.exists() {
|
||||
eprintln!("skipping: {} not present", path.display());
|
||||
return;
|
||||
}
|
||||
let pop = PostcodePopulation::load(path).expect("population parquet should load");
|
||||
|
||||
// Covers the whole of England & Wales (~1.37M unit postcodes).
|
||||
assert!(pop.by_postcode.len() > 1_000_000);
|
||||
// A residential postcode has a positive headcount...
|
||||
assert!(pop.for_postcode("AL1 1AG").is_some_and(|n| n > 0));
|
||||
// ...reachable from non-canonical input via normalize-on-load.
|
||||
assert_eq!(
|
||||
pop.for_postcode(&normalize_postcode("al1 1ag")),
|
||||
pop.for_postcode("AL1 1AG"),
|
||||
);
|
||||
// Postcodes with zero usual residents are absent from P001.
|
||||
assert_eq!(pop.for_postcode("EC1A 1BB"), None);
|
||||
}
|
||||
}
|
||||
|
|
@ -26,7 +26,9 @@ use rustc_hash::FxHashMap;
|
|||
use serde::Serialize;
|
||||
|
||||
use crate::consts::NAN_U16;
|
||||
use crate::data::area_crime_averages::{AreaCrimeAverages, AVG_YR_SUFFIX};
|
||||
use crate::data::spill::SpillVec;
|
||||
use crate::utils::{postcode_outcode, postcode_sector};
|
||||
|
||||
#[derive(Serialize, Clone)]
|
||||
pub struct RenovationEvent {
|
||||
|
|
@ -224,6 +226,109 @@ impl PropertyData {
|
|||
num_numeric: self.num_numeric,
|
||||
}
|
||||
}
|
||||
|
||||
/// Precompute mean headline crime rates nationally and per outcode / postcode
|
||||
/// sector.
|
||||
///
|
||||
/// Crime values are identical for every property in a postcode (the pipeline
|
||||
/// merges them on the postcode key), so each postcode is sampled once from
|
||||
/// its first row and property-weighted by its row count. All three scopes use
|
||||
/// the same exact property-weighted estimator over the same row universe as
|
||||
/// the per-selection mean, so the four numbers shown in a crime row (this
|
||||
/// selection / sector / outcode / nation) are directly comparable — without
|
||||
/// the upward bias of the histogram-bin national average.
|
||||
pub fn compute_area_crime_averages(&self) -> AreaCrimeAverages {
|
||||
// Crime headline columns are exactly the " (avg/yr)" features.
|
||||
let crime_indices: Vec<usize> = self
|
||||
.feature_names
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, name)| name.ends_with(AVG_YR_SUFFIX))
|
||||
.map(|(idx, _)| idx)
|
||||
.collect();
|
||||
if crime_indices.is_empty() {
|
||||
return AreaCrimeAverages::empty();
|
||||
}
|
||||
let crime_types: Vec<String> = crime_indices
|
||||
.iter()
|
||||
.map(|&idx| {
|
||||
self.feature_names[idx]
|
||||
.strip_suffix(AVG_YR_SUFFIX)
|
||||
.unwrap_or(&self.feature_names[idx])
|
||||
.to_string()
|
||||
})
|
||||
.collect();
|
||||
let n = crime_indices.len();
|
||||
|
||||
// (weighted value sum, weight) accumulators per crime type.
|
||||
let mut nat_sums = vec![0.0f64; n];
|
||||
let mut nat_weights = vec![0u64; n];
|
||||
let mut out_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
|
||||
let mut sec_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
|
||||
|
||||
for (key, rows) in &self.postcode_row_index {
|
||||
let Some(&first) = rows.first() else { continue };
|
||||
let count = rows.len() as u64;
|
||||
let postcode = self.postcode_interner.resolve(key);
|
||||
let outcode = postcode_outcode(postcode);
|
||||
let sector = postcode_sector(postcode);
|
||||
|
||||
for (j, &fi) in crime_indices.iter().enumerate() {
|
||||
// A NaN value is "no crime data for this postcode" — skip it so
|
||||
// it dilutes neither the sum nor the weight (a genuine gap, not
|
||||
// a zero), exactly as the global histogram excludes it.
|
||||
let value = self.get_feature(first as usize, fi);
|
||||
if !value.is_finite() {
|
||||
continue;
|
||||
}
|
||||
let weighted = value as f64 * count as f64;
|
||||
// National counts every postcode (the population the global mean
|
||||
// is built over); outcode/sector only when the postcode parses.
|
||||
nat_sums[j] += weighted;
|
||||
nat_weights[j] += count;
|
||||
if let Some(outcode) = outcode {
|
||||
let acc = out_acc
|
||||
.entry(outcode.to_string())
|
||||
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
|
||||
acc.0[j] += weighted;
|
||||
acc.1[j] += count;
|
||||
}
|
||||
if let Some(sector) = sector {
|
||||
let acc = sec_acc
|
||||
.entry(sector.to_string())
|
||||
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
|
||||
acc.0[j] += weighted;
|
||||
acc.1[j] += count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let means_of = |sums: &[f64], weights: &[u64]| -> Vec<f32> {
|
||||
sums.iter()
|
||||
.zip(weights.iter())
|
||||
.map(|(&sum, &weight)| {
|
||||
if weight == 0 {
|
||||
f32::NAN
|
||||
} else {
|
||||
(sum / weight as f64) as f32
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let finalize =
|
||||
|acc: FxHashMap<String, (Vec<f64>, Vec<u64>)>| -> FxHashMap<String, Vec<f32>> {
|
||||
acc.into_iter()
|
||||
.map(|(area, (sums, weights))| (area, means_of(&sums, &weights)))
|
||||
.collect()
|
||||
};
|
||||
|
||||
AreaCrimeAverages {
|
||||
crime_types,
|
||||
national: means_of(&nat_sums, &nat_weights),
|
||||
by_outcode: finalize(out_acc),
|
||||
by_sector: finalize(sec_acc),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -733,6 +733,138 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
|||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
// Education: Census 2021 TS067 highest-qualification breakdown. The
|
||||
// seven bands sum to 100% per neighbourhood (LSOA) and render as a
|
||||
// stacked composition (see STACKED_GROUPS["Neighbours"] in the
|
||||
// frontend), like the ethnicity and vote-share bars. Colloquial
|
||||
// labels stand in for the ONS "Level 1/2/3/4+" jargon.
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% No qualifications",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) with no qualifications",
|
||||
detail: "From the 2021 Census (TS067). Percentage of usual residents aged 16 and over in the neighbourhood (LSOA) who hold no formal qualifications.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Some GCSEs",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) whose highest qualification is roughly 1-4 GCSEs (Level 1)",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is around 1 to 4 GCSEs at grades 9-4 (A*-C), or entry-level/foundation qualifications. The ONS calls this 'Level 1 and entry level'.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Good GCSEs",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) whose highest qualification is 5+ GCSEs (Level 2)",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is roughly 5 or more GCSEs at grades 9-4 (A*-C), an intermediate apprenticeship, or equivalent. The ONS calls this 'Level 2'.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Apprenticeship",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) whose highest qualification is an apprenticeship",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is an apprenticeship.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% A-levels",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) whose highest qualification is A-levels (Level 3)",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is A-levels, AS-levels, T-levels, an advanced apprenticeship, or equivalent — typically studied after 16 and before a degree. The ONS calls this 'Level 3'.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Degree or higher",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) with a degree-level or higher qualification",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is degree level or above — a Bachelor's, Master's or PhD, foundation degree, HNC/HND, NVQ 4-5, or higher professional qualification. The census does not separate undergraduate from postgraduate degrees. The ONS calls this 'Level 4 or above'.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Other qualifications",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of residents (16+) with other qualifications, including vocational or overseas ones",
|
||||
detail: "From the 2021 Census (TS067). Highest qualification is classed as 'other' — vocational or professional qualifications not mapped to a UK level, and qualifications gained outside the UK.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
// Tenure: Census 2021 TS054 household-tenure breakdown. The three
|
||||
// shares sum to ~100% per neighbourhood (LSOA) and render as a
|
||||
// stacked composition (see STACKED_GROUPS["Neighbours"] in the
|
||||
// frontend), like the ethnicity, qualifications and vote-share bars.
|
||||
// Unlike those, the three shares are ALSO offered as individual
|
||||
// filters (they are not added to the display-only skip-list in
|
||||
// Filters.tsx), so users can target e.g. owner-occupier-heavy areas.
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Owner occupied",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of households that own their home, outright or with a mortgage",
|
||||
detail: "From the 2021 Census (TS054). Percentage of households in the neighbourhood (LSOA) that own their home outright, own it with a mortgage or loan, or hold it through shared ownership.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Social rent",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of households renting from a council or housing association",
|
||||
detail: "From the 2021 Census (TS054). Percentage of households in the neighbourhood (LSOA) renting from a local council or local authority, or from a housing association or other social landlord.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% Private rent",
|
||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||
step: 1.0,
|
||||
description: "Share of households renting privately or living rent-free",
|
||||
detail: "From the 2021 Census (TS054). Percentage of households in the neighbourhood (LSOA) renting from a private landlord or letting agency, plus the small share living rent-free.",
|
||||
source: "census-2021",
|
||||
prefix: "",
|
||||
suffix: "%",
|
||||
raw: false,
|
||||
absolute: false,
|
||||
}),
|
||||
Feature::Numeric(FeatureConfig {
|
||||
name: "% White",
|
||||
bounds: Bounds::Fixed {
|
||||
|
|
|
|||
|
|
@ -325,10 +325,21 @@ struct Cli {
|
|||
#[arg(long, env = "ACTUAL_LISTINGS_PATH")]
|
||||
actual_listings_path: PathBuf,
|
||||
|
||||
/// Path to a parquet of planned/pipeline development sites (MHCLG brownfield
|
||||
/// register + Homes England Land Hub) for the "new developments" layer.
|
||||
#[arg(long, env = "DEVELOPMENTS_PATH")]
|
||||
developments_path: PathBuf,
|
||||
|
||||
/// Path to the per-LSOA per-year crime parquet (display-only side table for the right pane).
|
||||
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
|
||||
crime_by_year_path: PathBuf,
|
||||
|
||||
/// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
|
||||
/// residents; display-only side table for the right pane). Optional: when
|
||||
/// absent or missing, the area pane simply omits the population figure.
|
||||
#[arg(long, env = "POPULATION_PATH")]
|
||||
population_path: Option<PathBuf>,
|
||||
|
||||
/// Google Maps API key for Street View metadata lookups
|
||||
#[arg(long, env = "GOOGLE_MAPS_API_KEY")]
|
||||
google_maps_api_key: String,
|
||||
|
|
@ -692,6 +703,18 @@ async fn main() -> anyhow::Result<()> {
|
|||
Arc::new(listings)
|
||||
};
|
||||
|
||||
let developments = {
|
||||
let path = &cli.developments_path;
|
||||
if !path.exists() {
|
||||
bail!("Development sites parquet not found: {}", path.display());
|
||||
}
|
||||
info!("Loading development sites from {}", path.display());
|
||||
let data = data::DevelopmentData::load(path)?;
|
||||
trim_allocator("development sites load");
|
||||
info!(rows = data.lat.len(), "Development sites loaded");
|
||||
Arc::new(data)
|
||||
};
|
||||
|
||||
let crime_by_year = {
|
||||
let path = &cli.crime_by_year_path;
|
||||
if !path.exists() {
|
||||
|
|
@ -702,6 +725,34 @@ async fn main() -> anyhow::Result<()> {
|
|||
Arc::new(data)
|
||||
};
|
||||
|
||||
let population = match &cli.population_path {
|
||||
Some(path) if path.exists() => {
|
||||
let data = data::PostcodePopulation::load(path)?;
|
||||
trim_allocator("postcode population load");
|
||||
Arc::new(data)
|
||||
}
|
||||
Some(path) => {
|
||||
tracing::warn!(
|
||||
"Population parquet not found at {}; area pane will omit population",
|
||||
path.display()
|
||||
);
|
||||
Arc::new(data::PostcodePopulation::empty())
|
||||
}
|
||||
None => Arc::new(data::PostcodePopulation::empty()),
|
||||
};
|
||||
|
||||
let area_crime_averages = {
|
||||
let data = property_data.compute_area_crime_averages();
|
||||
info!(
|
||||
outcodes = data.by_outcode.len(),
|
||||
sectors = data.by_sector.len(),
|
||||
crime_types = data.crime_types.len(),
|
||||
"Per-outcode/sector crime averages computed"
|
||||
);
|
||||
trim_allocator("area crime averages");
|
||||
Arc::new(data)
|
||||
};
|
||||
|
||||
let app_state = AppState {
|
||||
data: property_data,
|
||||
grid,
|
||||
|
|
@ -728,7 +779,10 @@ async fn main() -> anyhow::Result<()> {
|
|||
gemini_model: cli.gemini_model,
|
||||
travel_time_store,
|
||||
actual_listings,
|
||||
developments,
|
||||
crime_by_year,
|
||||
population,
|
||||
area_crime_averages,
|
||||
token_cache,
|
||||
superuser_token_cache,
|
||||
share_cache,
|
||||
|
|
@ -801,6 +855,10 @@ async fn main() -> anyhow::Result<()> {
|
|||
"/api/actual-listings",
|
||||
get(routes::get_actual_listings).layer(ConcurrencyLimitLayer::new(20)),
|
||||
)
|
||||
.route(
|
||||
"/api/developments",
|
||||
get(routes::get_developments).layer(ConcurrencyLimitLayer::new(20)),
|
||||
)
|
||||
.route(
|
||||
"/api/poi-categories",
|
||||
get(routes::get_poi_categories).layer(ConcurrencyLimitLayer::new(20)),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
mod actual_listings;
|
||||
mod ai_filters;
|
||||
mod checkout;
|
||||
mod developments;
|
||||
mod export;
|
||||
mod features;
|
||||
mod filter_counts;
|
||||
|
|
@ -34,6 +35,7 @@ pub(crate) mod travel_time;
|
|||
pub use actual_listings::get_actual_listings;
|
||||
pub use ai_filters::{build_system_prompt, post_ai_filters};
|
||||
pub use checkout::post_checkout;
|
||||
pub use developments::get_developments;
|
||||
pub use export::get_export;
|
||||
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
|
||||
pub use filter_counts::get_filter_counts;
|
||||
|
|
|
|||
70
server-rs/src/routes/developments.rs
Normal file
70
server-rs/src/routes/developments.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::response::{IntoResponse, Json, Response};
|
||||
use axum::Extension;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::info;
|
||||
|
||||
use crate::auth::OptionalUser;
|
||||
use crate::data::DevelopmentSite;
|
||||
use crate::licensing::{check_license_bounds, resolve_share_code};
|
||||
use crate::parsing::require_bounds;
|
||||
use crate::state::SharedState;
|
||||
|
||||
/// Hard cap on sites returned per viewport. Biggest schemes (by dwelling count)
|
||||
/// are kept when a dense viewport exceeds this; `truncated` flags the clamp.
|
||||
const DEVELOPMENTS_LIMIT: usize = 4000;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DevelopmentsParams {
|
||||
bounds: Option<String>,
|
||||
/// Share-link code; grants bbox-scoped access for unlicensed users.
|
||||
share: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct DevelopmentsResponse {
|
||||
pub developments: Vec<DevelopmentSite>,
|
||||
pub total: usize,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
/// Forward-looking "where new homes are coming" layer: planned/pipeline
|
||||
/// development sites (MHCLG Brownfield Land register + Homes England Land Hub),
|
||||
/// served as points within a viewport. Public OGL data, but still gated by the
|
||||
/// normal demo/licence bounds check so unlicensed users only see their free zone.
|
||||
pub async fn get_developments(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||
Query(params): Query<DevelopmentsParams>,
|
||||
) -> Result<Json<DevelopmentsResponse>, Response> {
|
||||
let state = shared.load_state();
|
||||
|
||||
let (south, west, north, east) =
|
||||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||
|
||||
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||
check_license_bounds(
|
||||
&user.0,
|
||||
(south, west, north, east),
|
||||
geo.free_zone,
|
||||
share_bounds,
|
||||
)?;
|
||||
|
||||
let developments = state.developments.clone();
|
||||
let (sites, total, truncated) =
|
||||
developments.query_bounds(south, west, north, east, DEVELOPMENTS_LIMIT);
|
||||
|
||||
info!(
|
||||
results = sites.len(),
|
||||
total, truncated, "GET /api/developments"
|
||||
);
|
||||
|
||||
Ok(Json(DevelopmentsResponse {
|
||||
developments: sites,
|
||||
total,
|
||||
truncated,
|
||||
}))
|
||||
}
|
||||
|
|
@ -80,6 +80,24 @@ pub struct CrimeYearStats {
|
|||
pub points: Vec<CrimeYearPoint>,
|
||||
}
|
||||
|
||||
/// Average headline crime rate (avg/yr) for one crime type across the
|
||||
/// selection's outcode and postcode sector. Comparable to the national average
|
||||
/// shown per metric in the right pane.
|
||||
#[derive(Serialize)]
|
||||
pub struct CrimeAreaAverage {
|
||||
/// Crime type, bare (e.g. "Burglary"). Matches `CrimeYearStats.name`.
|
||||
pub name: String,
|
||||
/// Exact national mean (avg/yr) — the frontend prefers this over the
|
||||
/// histogram-bin national average for crime so all four numbers in the row
|
||||
/// share one estimator.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub national: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub outcode: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sector: Option<f32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct FilterExclusion {
|
||||
pub name: String,
|
||||
|
|
@ -135,8 +153,25 @@ pub struct HexagonStatsResponse {
|
|||
/// the client captions the data as stale.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub crime_latest_year: Option<i32>,
|
||||
/// Outward code (e.g. "E14") of the selection's central postcode, present
|
||||
/// only when outcode crime averages are available for it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub crime_outcode: Option<String>,
|
||||
/// Postcode sector (e.g. "E14 2") of the selection's central postcode,
|
||||
/// present only when sector crime averages are available for it.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub crime_sector: Option<String>,
|
||||
/// Per-crime-type average rates across the central postcode's outcode and
|
||||
/// sector, shown alongside the national average for each crime metric.
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub crime_area_averages: Vec<CrimeAreaAverage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub central_postcode: Option<String>,
|
||||
/// Total usual residents (ONS Census 2021) living across the distinct
|
||||
/// postcodes in this selection. Display-only and independent of active
|
||||
/// filters; absent when no population data covers the selection.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub population: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub filter_exclusions: Vec<FilterExclusion>,
|
||||
}
|
||||
|
|
@ -657,6 +692,33 @@ pub async fn get_hexagon_stats(
|
|||
stats::crime_latest_available_year(&state.crime_by_year)
|
||||
};
|
||||
|
||||
let (crime_outcode, crime_sector, crime_area_averages) = stats::area_crime_averages_for(
|
||||
central_postcode.as_deref(),
|
||||
&state.area_crime_averages,
|
||||
fields_specified,
|
||||
&field_set,
|
||||
);
|
||||
|
||||
// Sum usual residents across the distinct postcodes covered by the
|
||||
// hexagon. Computed over `area_rows` (all properties in the cell), not
|
||||
// the filter-matching subset, so toggling filters never changes it —
|
||||
// population is an attribute of the area, like the council-house count.
|
||||
let population = {
|
||||
let mut seen: HashSet<&str> = HashSet::new();
|
||||
let mut total: u64 = 0;
|
||||
let mut found = false;
|
||||
for &row in &area_rows {
|
||||
let pc = state.data.postcode(row);
|
||||
if seen.insert(pc) {
|
||||
if let Some(p) = state.population.for_postcode(pc) {
|
||||
total += p as u64;
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
found.then(|| total.min(u32::MAX as u64) as u32)
|
||||
};
|
||||
|
||||
Ok(HexagonStatsResponse {
|
||||
count: total_count,
|
||||
numeric_features,
|
||||
|
|
@ -664,7 +726,11 @@ pub async fn get_hexagon_stats(
|
|||
price_history,
|
||||
crime_by_year,
|
||||
crime_latest_year,
|
||||
crime_outcode,
|
||||
crime_sector,
|
||||
crime_area_averages,
|
||||
central_postcode,
|
||||
population,
|
||||
filter_exclusions,
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -217,6 +217,16 @@ pub async fn get_postcode_stats(
|
|||
stats::crime_latest_available_year(&state.crime_by_year)
|
||||
};
|
||||
|
||||
let (crime_outcode, crime_sector, crime_area_averages) = stats::area_crime_averages_for(
|
||||
Some(postcode_str.as_str()),
|
||||
&state.area_crime_averages,
|
||||
fields_specified,
|
||||
&field_set,
|
||||
);
|
||||
|
||||
// Usual residents (Census 2021) for this postcode. Display-only.
|
||||
let population = state.population.for_postcode(&postcode_str);
|
||||
|
||||
Ok(HexagonStatsResponse {
|
||||
count: total_count,
|
||||
numeric_features,
|
||||
|
|
@ -224,7 +234,11 @@ pub async fn get_postcode_stats(
|
|||
price_history,
|
||||
crime_by_year,
|
||||
crime_latest_year,
|
||||
crime_outcode,
|
||||
crime_sector,
|
||||
crime_area_averages,
|
||||
central_postcode: None,
|
||||
population,
|
||||
filter_exclusions,
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -5,12 +5,14 @@ use rustc_hash::FxHashMap;
|
|||
use tracing::error;
|
||||
|
||||
use crate::consts::PRICE_HISTORY_POINTS_LIMIT;
|
||||
use crate::data::area_crime_averages::AreaCrimeAverages;
|
||||
use crate::data::crime_by_year::CrimeByYearData;
|
||||
use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
|
||||
use crate::utils::{postcode_outcode, postcode_sector};
|
||||
|
||||
use super::hexagon_stats::{
|
||||
CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats, NumericFeatureStats,
|
||||
PricePoint,
|
||||
CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats,
|
||||
NumericFeatureStats, PricePoint,
|
||||
};
|
||||
|
||||
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
|
||||
|
|
@ -401,6 +403,76 @@ pub fn crime_latest_available_year(crime_by_year: &CrimeByYearData) -> Option<i3
|
|||
crime_by_year.years_by_type.iter().flatten().copied().max()
|
||||
}
|
||||
|
||||
/// Per-crime-type national/outcode/sector averages for a selection, keyed off
|
||||
/// its central postcode. Returns `(outcode, sector, per_type_averages)` where
|
||||
/// the outcode/sector strings are present only when matching averages exist.
|
||||
///
|
||||
/// Honours the same `fields` filtering as [`compute_crime_by_year`]: when fields
|
||||
/// are specified, only the requested crime types are emitted, and a type is
|
||||
/// omitted only when none of its national/outcode/sector averages are known.
|
||||
pub fn area_crime_averages_for(
|
||||
central_postcode: Option<&str>,
|
||||
averages: &AreaCrimeAverages,
|
||||
fields_specified: bool,
|
||||
field_set: &HashSet<String>,
|
||||
) -> (Option<String>, Option<String>, Vec<CrimeAreaAverage>) {
|
||||
let none = || (None, None, Vec::new());
|
||||
let Some(postcode) = central_postcode else {
|
||||
return none();
|
||||
};
|
||||
if averages.crime_types.is_empty() {
|
||||
return none();
|
||||
}
|
||||
|
||||
let outcode = postcode_outcode(postcode);
|
||||
let sector = postcode_sector(postcode);
|
||||
let outcode_means = outcode.and_then(|code| averages.by_outcode.get(code));
|
||||
let sector_means = sector.and_then(|code| averages.by_sector.get(code));
|
||||
// National is always available, so we still emit it (to override the
|
||||
// histogram national average for crime) even when this postcode's outcode
|
||||
// and sector both lack precomputed data.
|
||||
|
||||
let finite_at = |means: Option<&Vec<f32>>, idx: usize| -> Option<f32> {
|
||||
means
|
||||
.and_then(|m| m.get(idx).copied())
|
||||
.filter(|v| v.is_finite())
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (idx, name) in averages.crime_types.iter().enumerate() {
|
||||
// Crime types are bare here ("Burglary"); requested fields may carry the
|
||||
// " (avg/yr)" suffix — accept either form, like compute_crime_by_year.
|
||||
if fields_specified {
|
||||
let with_suffix = format!("{name} (avg/yr)");
|
||||
if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let national_val = finite_at(Some(&averages.national), idx);
|
||||
let outcode_val = finite_at(outcode_means, idx);
|
||||
let sector_val = finite_at(sector_means, idx);
|
||||
if national_val.is_none() && outcode_val.is_none() && sector_val.is_none() {
|
||||
continue;
|
||||
}
|
||||
out.push(CrimeAreaAverage {
|
||||
name: name.clone(),
|
||||
national: national_val,
|
||||
outcode: outcode_val,
|
||||
sector: sector_val,
|
||||
});
|
||||
}
|
||||
|
||||
(
|
||||
outcode
|
||||
.filter(|_| outcode_means.is_some())
|
||||
.map(str::to_string),
|
||||
sector
|
||||
.filter(|_| sector_means.is_some())
|
||||
.map(str::to_string),
|
||||
out,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn compute_poi_feature_stats(
|
||||
matching_rows: &[usize],
|
||||
poi_metrics: &PostcodePoiMetrics,
|
||||
|
|
@ -516,4 +588,70 @@ mod tests {
|
|||
let data = enum_data(&[0, 1]);
|
||||
assert!(compute_enum_feature_counts(&[0, 1], &data, 1).is_none());
|
||||
}
|
||||
|
||||
fn sample_averages() -> AreaCrimeAverages {
|
||||
let mut by_outcode = rustc_hash::FxHashMap::default();
|
||||
by_outcode.insert("E14".to_string(), vec![10.0, f32::NAN]);
|
||||
let mut by_sector = rustc_hash::FxHashMap::default();
|
||||
by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
|
||||
AreaCrimeAverages {
|
||||
crime_types: vec!["Burglary".to_string(), "Robbery".to_string()],
|
||||
national: vec![8.0, 6.0],
|
||||
by_outcode,
|
||||
by_sector,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn area_crime_averages_populate_national_outcode_sector() {
|
||||
let avgs = sample_averages();
|
||||
let (outcode, sector, out) =
|
||||
area_crime_averages_for(Some("E14 2DG"), &avgs, false, &HashSet::new());
|
||||
assert_eq!(outcode.as_deref(), Some("E14"));
|
||||
assert_eq!(sector.as_deref(), Some("E14 2"));
|
||||
assert_eq!(out.len(), 2);
|
||||
|
||||
let burglary = out.iter().find(|c| c.name == "Burglary").unwrap();
|
||||
assert_eq!(burglary.national, Some(8.0));
|
||||
assert_eq!(burglary.outcode, Some(10.0));
|
||||
assert_eq!(burglary.sector, Some(5.0));
|
||||
|
||||
let robbery = out.iter().find(|c| c.name == "Robbery").unwrap();
|
||||
assert_eq!(robbery.national, Some(6.0));
|
||||
// The outcode value was NaN — dropped to None; the sector value is finite.
|
||||
assert_eq!(robbery.outcode, None);
|
||||
assert_eq!(robbery.sector, Some(7.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn area_crime_averages_respect_fields_filter() {
|
||||
let avgs = sample_averages();
|
||||
// The suffixed feature-name form is accepted, like compute_crime_by_year.
|
||||
let fields: HashSet<String> = ["Burglary (avg/yr)".to_string()].into_iter().collect();
|
||||
let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].name, "Burglary");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn area_crime_averages_unknown_area_still_emits_national() {
|
||||
let avgs = sample_averages();
|
||||
let (outcode, sector, out) =
|
||||
area_crime_averages_for(Some("ZZ9 9ZZ"), &avgs, false, &HashSet::new());
|
||||
// The area is absent from both maps: no codes, but national still applies.
|
||||
assert!(outcode.is_none());
|
||||
assert!(sector.is_none());
|
||||
assert_eq!(out.len(), 2);
|
||||
assert!(out
|
||||
.iter()
|
||||
.all(|c| c.outcode.is_none() && c.sector.is_none()));
|
||||
assert_eq!(out[0].national, Some(8.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn area_crime_averages_none_without_postcode() {
|
||||
let avgs = sample_averages();
|
||||
let (outcode, sector, out) = area_crime_averages_for(None, &avgs, false, &HashSet::new());
|
||||
assert!(outcode.is_none() && sector.is_none() && out.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ use rustc_hash::FxHashMap;
|
|||
use crate::auth::TokenCache;
|
||||
use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
|
||||
use crate::data::{
|
||||
ActualListingData, CrimeByYearData, OutcodeData, POICategoryGroup, POIData, PlaceData,
|
||||
PostcodeData, PropertyData, TravelTimeStore,
|
||||
ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData,
|
||||
POICategoryGroup, POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData,
|
||||
TravelTimeStore,
|
||||
};
|
||||
use crate::licensing::ShareBoundsCache;
|
||||
use crate::pocketbase::SuperuserTokenCache;
|
||||
|
|
@ -46,8 +47,17 @@ pub struct AppState {
|
|||
pub travel_time_store: Arc<TravelTimeStore>,
|
||||
/// Real-world listings (e.g. Rightmove / Zoopla data) loaded from ACTUAL_LISTINGS_PATH.
|
||||
pub actual_listings: Arc<ActualListingData>,
|
||||
/// Planned/pipeline development sites (brownfield register + Homes England)
|
||||
/// loaded from the required DEVELOPMENTS_PATH.
|
||||
pub developments: Arc<DevelopmentData>,
|
||||
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
|
||||
pub crime_by_year: Arc<CrimeByYearData>,
|
||||
/// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
|
||||
/// right pane. Display-only — never filterable. Empty when no data is loaded.
|
||||
pub population: Arc<PostcodePopulation>,
|
||||
/// Precomputed per-outcode and per-postcode-sector average crime rates,
|
||||
/// shown in the right pane alongside the national average for each metric.
|
||||
pub area_crime_averages: Arc<AreaCrimeAverages>,
|
||||
/// Token validation cache (60s TTL)
|
||||
pub token_cache: Arc<TokenCache>,
|
||||
/// Cached PocketBase superuser token (10min TTL) to avoid rate-limiting
|
||||
|
|
@ -99,8 +109,8 @@ impl AppState {
|
|||
use std::time::Duration;
|
||||
|
||||
use crate::data::{
|
||||
ActualListingData, CrimeByYearData, OutcodeData, POIData, PlaceData, PostcodeData,
|
||||
PropertyData, TravelTimeStore,
|
||||
ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData,
|
||||
POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData, TravelTimeStore,
|
||||
};
|
||||
use crate::utils::InternedColumn;
|
||||
|
||||
|
|
@ -161,12 +171,15 @@ impl AppState {
|
|||
poi_filter_feature_data: Vec::new(),
|
||||
grid: GridIndex::build(&[], &[], 0.01),
|
||||
}),
|
||||
developments: Arc::new(DevelopmentData::empty()),
|
||||
crime_by_year: Arc::new(CrimeByYearData {
|
||||
crime_types: Vec::new(),
|
||||
years_by_type: Vec::new(),
|
||||
series_by_postcode: FxHashMap::default(),
|
||||
covered_years_by_postcode: FxHashMap::default(),
|
||||
}),
|
||||
population: Arc::new(PostcodePopulation::empty()),
|
||||
area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
|
||||
token_cache: Arc::new(TokenCache::new()),
|
||||
superuser_token_cache: Arc::new(SuperuserTokenCache::new()),
|
||||
share_cache: Arc::new(ShareBoundsCache::new()),
|
||||
|
|
|
|||
|
|
@ -20,3 +20,44 @@ pub fn normalize_postcode(raw: &str) -> String {
|
|||
upper
|
||||
}
|
||||
}
|
||||
|
||||
/// Outward code (outcode) of a normalized postcode: the part before the space.
|
||||
/// e.g. "E14 2DG" → Some("E14"). Returns `None` when the input has no space
|
||||
/// (already-short or malformed — `normalize_postcode` only inserts a space for
|
||||
/// inputs of length ≥ 5).
|
||||
pub fn postcode_outcode(postcode: &str) -> Option<&str> {
|
||||
postcode.split_once(' ').map(|(outward, _)| outward)
|
||||
}
|
||||
|
||||
/// Postcode sector of a normalized postcode: the outward code, the space, and
|
||||
/// the first character of the inward code. e.g. "E14 2DG" → Some("E14 2").
|
||||
/// Returns `None` when there is no space or no inward code.
|
||||
pub fn postcode_sector(postcode: &str) -> Option<&str> {
|
||||
let space = postcode.find(' ')?;
|
||||
let first = postcode[space + 1..].chars().next()?;
|
||||
Some(&postcode[..space + 1 + first.len_utf8()])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn outcode_and_sector_split() {
|
||||
assert_eq!(postcode_outcode("E14 2DG"), Some("E14"));
|
||||
assert_eq!(postcode_sector("E14 2DG"), Some("E14 2"));
|
||||
assert_eq!(postcode_outcode("EC1A 1BB"), Some("EC1A"));
|
||||
assert_eq!(postcode_sector("EC1A 1BB"), Some("EC1A 1"));
|
||||
// Single-letter area, single-digit district.
|
||||
assert_eq!(postcode_outcode("M1 1AE"), Some("M1"));
|
||||
assert_eq!(postcode_sector("M1 1AE"), Some("M1 1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcode_and_sector_reject_malformed() {
|
||||
assert_eq!(postcode_outcode("E14"), None);
|
||||
assert_eq!(postcode_sector("E14"), None);
|
||||
assert_eq!(postcode_outcode(""), None);
|
||||
assert_eq!(postcode_sector("E14 "), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue