andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Small fixes

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-14 14:52:44 +01:00
parent 54fbcb1ea6
commit 083f8a982e
24 changed files with 1505 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

23
server-rs/src/data.rs
View file

@ -6,6 +6,29 @@ mod postcodes;
mod property;
pub mod travel_time;
/// Apostrophe-like code points that should be elided (not treated as word breaks) when
/// normalizing search text. Keyboards, autocorrect and copy-paste all produce different glyphs
/// for what users mean as an apostrophe; treating any of them as a separator turns "King's Cross"
/// into the tokens `king s cross` and breaks matching. Covers the straight ASCII apostrophe, the
/// typographic left/right single quotes, grave/acute accents, the modifier-letter apostrophes,
/// prime, and the fullwidth apostrophe.
pub(crate) fn is_apostrophe(ch: char) -> bool {
matches!(
ch,
'\u{0027}' // ' APOSTROPHE
| '\u{0060}' // ` GRAVE ACCENT
| '\u{00B4}' // ´ ACUTE ACCENT
| '\u{02B9}' // ʹ MODIFIER LETTER PRIME
| '\u{02BB}' // ʻ MODIFIER LETTER TURNED COMMA (ʻokina)
| '\u{02BC}' // ʼ MODIFIER LETTER APOSTROPHE
| '\u{2018}' // ' LEFT SINGLE QUOTATION MARK
| '\u{2019}' // ' RIGHT SINGLE QUOTATION MARK
| '\u{201B}' // ‛ SINGLE HIGH-REVERSED-9 QUOTATION MARK
| '\u{2032}' // ′ PRIME
| '\u{FF07}' // ＇ FULLWIDTH APOSTROPHE
)
}
fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String {
if let Some(message) = payload.downcast_ref::<&'static str>() {
(*message).to_string()

22
server-rs/src/data/places.rs
View file

@ -161,7 +161,7 @@ pub fn normalize_search_text(text: &str) -> String {
let mut last_was_space = true;
for ch in text.chars() {
if ch == '\'' || ch == '’' || ch == '`' {
if super::is_apostrophe(ch) {
continue;
}
@ -769,6 +769,26 @@ impl PlaceData {
mod tests {
use super::*;
#[test]
fn normalize_search_text_elides_every_apostrophe_variant() {
// Whatever glyph the keyboard / autocorrect / paste produced for the apostrophe, the
// normalized form must be identical — otherwise "King's Cross" tokenizes as `king s cross`
// and matching breaks. See is_apostrophe for the full set.
let expected = "kings cross";
for q in [
"King's Cross", // U+0027 straight
"King’s Cross", // U+2019 right single quote
"King‘s Cross", // U+2018 left single quote
"King´s Cross", // U+00B4 acute accent
"Kingʼs Cross", // U+02BC modifier letter apostrophe
"King′s Cross", // U+2032 prime
"King`s Cross", // U+0060 grave accent
"Kingʻs Cross", // U+02BB modifier letter turned comma
] {
assert_eq!(normalize_search_text(q), expected, "failed for {q:?}");
}
}
#[test]
fn place_index_tokens_dedup_and_min_length() {
// "a" is too short; aliases split on " | ".

20
server-rs/src/data/property/address_search.rs
View file

@ -37,7 +37,7 @@ fn tokenize_address_text(text: &str) -> Vec<String> {
for ch in text.chars() {
if ch.is_ascii_alphanumeric() {
current.push(ch.to_ascii_lowercase());
} else if matches!(ch, '\'' | '’' | '`') {
} else if crate::data::is_apostrophe(ch) {
continue;
} else if !current.is_empty() {
tokens.push(std::mem::take(&mut current));
@ -784,6 +784,24 @@ impl PropertyData {
mod tests {
use super::*;
#[test]
fn tokenize_address_text_elides_every_apostrophe_variant() {
// The query and the index both run through this tokenizer, so any apostrophe glyph must
// join the word ("O'Brien" -> "obrien") rather than split it; otherwise the same address
// tokenizes differently depending on which quote glyph was typed.
let expected = vec!["obrien".to_string(), "road".to_string()];
for addr in [
"O'Brien Road", // U+0027 straight
"O’Brien Road", // U+2019 right single quote
"O‘Brien Road", // U+2018 left single quote
"O´Brien Road", // U+00B4 acute accent
"OʼBrien Road", // U+02BC modifier letter apostrophe
"O′Brien Road", // U+2032 prime
] {
assert_eq!(tokenize_address_text(addr), expected, "failed for {addr:?}");
}
}
#[test]
fn full_postcode_detection_accepts_common_formats() {
assert!(is_full_postcode_compact("SW1A1AA"));

149
server-rs/src/features.rs
View file

@ -31,10 +31,17 @@ pub struct FeatureConfig {
pub absolute: bool,
}
/// Features whose histogram bins should be exactly 1 unit wide (one per integer).
/// p1/p99 are snapped to integer boundaries before binning.
/// Explicit override for features that should use integer-width histogram bins
/// (one bin per integer, p1/p99 snapped to integer boundaries) even if they
/// don't match the automatic rule in [`has_integer_bins`]. Most integer count
/// features are detected automatically from their config.
pub const INTEGER_BIN_FEATURES: &[&str] = &["Number of bedrooms & living rooms"];
/// Largest p1–p99 span (in integer units) for which one-bin-per-integer
/// histograms stay reasonable. Above this, percentile binning is used instead so
/// wide-range features (years, percentages, prices) don't allocate one bin each.
const MAX_INTEGER_BIN_RANGE: f32 = 16.0;
pub struct EnumFeatureConfig {
pub name: &'static str,
/// If set, values are presented in this order instead of alphabetical.
@ -473,11 +480,11 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
high: 98.0,
},
step: 1.0,
description: "Aggregate of serious crime categories per year",
detail: "Sum of violence, robbery, burglary, and weapons possession per year near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a per-resident risk: busy commercial centres rank high however few people live there. Averaged over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
description: "Relative density of serious crime categories near the postcode",
detail: "Combined density of violence, robbery, burglary, and weapons possession near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -488,11 +495,11 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
high: 98.0,
},
step: 1.0,
description: "Aggregate of minor crime categories per year",
detail: "Sum of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime per year near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a per-resident risk: busy commercial centres rank high however few people live there. Averaged over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
description: "Relative density of minor crime categories near the postcode",
detail: "Combined density of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -507,7 +514,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of violence and sexual offences per year near the postcode, from police.uk street-level crime data. Includes assault, harassment, and sexual offences.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -522,7 +529,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of burglary offences per year near the postcode, from police.uk street-level crime data. Includes residential and commercial burglary.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -537,7 +544,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of robbery offences per year near the postcode, from police.uk street-level crime data. Robbery involves theft with force or threat of force.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -552,7 +559,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of vehicle crime incidents per year near the postcode, from police.uk street-level crime data. Includes theft of and from vehicles.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -567,7 +574,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of anti-social behaviour incidents per year near the postcode, from police.uk street-level crime data. Includes nuisance, environmental, and personal anti-social behaviour.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -582,7 +589,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of criminal damage and arson incidents per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -597,7 +604,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of 'other theft' offences per year near the postcode, from police.uk street-level crime data. Includes theft not classified under burglary, vehicle crime, shoplifting, or bicycle theft.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -612,7 +619,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of theft from the person offences per year near the postcode, from police.uk street-level crime data. Includes pickpocketing and bag snatching without force.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -627,7 +634,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of shoplifting offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -642,7 +649,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of bicycle theft offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -657,7 +664,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of drug offences per year near the postcode, from police.uk street-level crime data. Includes possession and trafficking offences.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -672,7 +679,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of possession of weapons offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -687,7 +694,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of public order offences per year near the postcode, from police.uk street-level crime data. Includes causing fear, alarm, or distress.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -702,7 +709,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
detail: "Average number of other crime offences per year near the postcode, from police.uk street-level crime data. A catch-all category for offences not classified elsewhere.",
source: "crime",
prefix: "",
suffix: "/yr",
suffix: "",
raw: false,
absolute: false,
}),
@ -734,7 +741,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 1.0,
description: "Percentage of population identifying as White",
detail: "From the 2021 Census. Percentage of the local authority population identifying as White (English, Welsh, Scottish, Northern Irish, British, Irish, Gypsy or Irish Traveller, Roma, or any other White background).",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as White (English, Welsh, Scottish, Northern Irish, British, Irish, Gypsy or Irish Traveller, Roma, or any other White background).",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -749,7 +756,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as South Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Indian, Pakistani, Bangladeshi, or any other Asian background.",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as Indian, Pakistani, or Bangladeshi.",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -764,7 +771,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as Black",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Black, Black British, Caribbean, or African.",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as Black, Black British, Caribbean, or African.",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -779,7 +786,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as East Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Chinese.",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as Chinese.",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -794,7 +801,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as Southeast Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as another (non-Chinese) East or Southeast Asian background, e.g. Vietnamese, Filipino, or Thai.",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as another (non-Chinese) East or Southeast Asian background, e.g. Vietnamese, Filipino, or Thai.",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -809,7 +816,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as Mixed or Multiple ethnic groups",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Mixed or Multiple ethnic groups (White and Black Caribbean, White and Black African, White and Asian, or any other Mixed or Multiple background).",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as Mixed or Multiple ethnic groups (White and Black Caribbean, White and Black African, White and Asian, or any other Mixed or Multiple background).",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -824,7 +831,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 0.1,
description: "Percentage of population identifying as Other ethnic group",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Other ethnic group (Arab or any other ethnic group not covered by the main categories).",
detail: "From the 2021 Census. Percentage of the local neighbourhood (LSOA) population identifying as Other ethnic group (Arab or any other ethnic group not covered by the main categories).",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -997,9 +1004,48 @@ pub fn order_for(name: &str) -> Option<&'static [&'static str]> {
.flatten()
}
/// Whether this feature should use integer-width histogram bins.
/// Whether this feature should use integer-width histogram bins: one bin per
/// integer value with p1/p99 snapped to integer boundaries. This keeps the
/// histogram bars aligned 1:1 with the integer axis labels in the UI (otherwise
/// fractional percentiles and non-unit bin widths make the lit bar drift away
/// from its label, e.g. a value of 3 landing under the "2" tick).
///
/// Applies to small-range integer-valued count features (school catchments,
/// bedrooms, …) plus the dynamic POI count features. Wide-range or fractional
/// features (years, percentages, crime averages) keep percentile binning.
pub fn has_integer_bins(name: &str) -> bool {
INTEGER_BIN_FEATURES.contains(&name) || dynamic_poi_count_radius(name).is_some()
if INTEGER_BIN_FEATURES.contains(&name) || dynamic_poi_count_radius(name).is_some() {
return true;
}
matches!(numeric_config_for(name), Some(c) if is_integer_count_feature(c))
}
/// True for a numeric feature whose values are small integer counts: unit step
/// with fixed integer bounds spanning a small enough range for one bin per value.
fn is_integer_count_feature(config: &FeatureConfig) -> bool {
if config.step != 1.0 {
return false;
}
match config.bounds {
Bounds::Fixed { min, max } => {
min.fract() == 0.0
&& max.fract() == 0.0
&& max > min
&& (max - min) <= MAX_INTEGER_BIN_RANGE
}
Bounds::Percentile { .. } => false,
}
}
/// Look up the static numeric feature config by exact name.
fn numeric_config_for(name: &str) -> Option<&'static FeatureConfig> {
FEATURE_GROUPS
.iter()
.flat_map(|group| group.features.iter())
.find_map(|feature| match feature {
Feature::Numeric(c) if c.name == name => Some(c),
_ => None,
})
}
/// Look up the Bounds config for a numeric feature by name.
@ -1084,3 +1130,44 @@ pub const POI_GROUP_ORDER: &[&str] = &[
"Services",
"Shops",
];
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn integer_bins_cover_small_integer_count_features() {
// Small integer count features must get one bin per integer so the
// histogram bars line up with their integer axis labels.
for name in [
"Good+ primary school catchments",
"Good+ secondary school catchments",
"Outstanding primary school catchments",
"Outstanding secondary school catchments",
"Number of bedrooms & living rooms",
] {
assert!(has_integer_bins(name), "{name} should use integer bins");
}
}
#[test]
fn integer_bins_exclude_wide_range_and_fractional_features() {
// step == 1.0 but wide-range (would explode bin count) or fractional
// (averages/percentiles) features must keep percentile binning.
for name in [
"Construction year", // Fixed but spans ~2000 years
"Date of last transaction", // Fixed, fractional years
"Income Score", // 0..100 percentile
"% White", // 0..100 percentage
"Noise (dB)", // 50..80, range > threshold
"Serious crime (avg/yr)", // Percentile bounds, fractional
"Interior height (m)", // step 0.1
"Estimated current price", // step 10000
] {
assert!(
!has_integer_bins(name),
"{name} should not use integer bins"
);
}
}
}

29
server-rs/src/routes/export.rs
View file

@ -440,6 +440,11 @@ pub async fn get_export(
.map_err(|err| (StatusCode::BAD_REQUEST, err).into_response())?
};
let has_poi_filters = !parsed_poi_filters.is_empty();
// The active filters decide which feature columns land on the "Selected"
// sheet. We keep them for column selection in both modes; they are only
// *applied* as row filters in bounds mode (in list mode the user picks the
// postcodes explicitly, so rows are never filtered).
let column_filters_str = params.filters.clone();
let filters_str = if is_postcode_mode {
None
} else {
@ -653,7 +658,7 @@ pub async fn get_export(
};
// Determine column order: filter features first, then remaining
let filter_feature_names = extract_filter_feature_names(filters_str.as_deref());
let filter_feature_names = extract_filter_feature_names(column_filters_str.as_deref());
let field_indices = parse_field_indices_with_poi(
fields_str.as_deref(),
@ -841,18 +846,16 @@ pub async fn get_export(
frontend_params
);
// Bounds mode: two sheets — "Selected" (filter features with link + screenshot)
// and "All Data" (all features).
// List mode: single sheet "Postcodes" with all data, no link or screenshot
// (the supplied list isn't tied to a map view).
let sheet_configs: Vec<(&str, &[usize], bool)> = if postcode_list_entries.is_some() {
vec![("Postcodes", &all_feature_indices, false)]
} else {
vec![
("Selected", &filter_feature_indices, true),
("All Data", &all_feature_indices, false),
]
};
// Two sheets in both modes: "Selected" (just the features behind the
// active filters) and "All Data" (every feature). The Selected sheet
// carries the dashboard link + screenshot only in bounds mode, where the
// export is tied to a map view; in list mode the user picked the
// postcodes explicitly, so there's no map view to link or screenshot.
let is_list_mode = postcode_list_entries.is_some();
let sheet_configs: Vec<(&str, &[usize], bool)> = vec![
("Selected", &filter_feature_indices, !is_list_mode),
("All Data", &all_feature_indices, false),
];
for (sheet_name, feat_indices, include_header) in &sheet_configs {
let sheet = workbook.add_worksheet();