fix bin silliness

This commit is contained in:
Andras Schmelczer 2026-07-14 14:10:47 +01:00
parent d1faad314b
commit d78a301028

View file

@ -8,6 +8,7 @@ use crate::consts::{AREA_PRICE_HISTORY_POINTS_LIMIT, PRICE_HISTORY_POINTS_LIMIT}
use crate::data::area_crime_averages::AreaCrimeAverages; use crate::data::area_crime_averages::AreaCrimeAverages;
use crate::data::crime_by_year::CrimeByYearData; use crate::data::crime_by_year::CrimeByYearData;
use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData}; use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
use crate::features::has_integer_bins;
use crate::utils::{postcode_outcode, postcode_sector}; use crate::utils::{postcode_outcode, postcode_sector};
use super::hexagon_stats::{ use super::hexagon_stats::{
@ -146,6 +147,10 @@ enum FeatureAccum {
num_bins: usize, num_bins: usize,
global_min: f32, global_min: f32,
global_max: f32, global_max: f32,
/// Integer-count feature: values are u16-quantized so a genuine integer N
/// can decode to N-epsilon; round to the nearest integer before binning so
/// the lit bar lands under its own axis label instead of one bucket left.
integer_bins: bool,
}, },
/// Enum: count occurrences per variant index. /// Enum: count occurrences per variant index.
Enum { value_counts: Vec<u64> }, Enum { value_counts: Vec<u64> },
@ -202,6 +207,7 @@ pub fn compute_feature_stats(
num_bins, num_bins,
global_min: global_hist.min, global_min: global_hist.min,
global_max: global_hist.max, global_max: global_hist.max,
integer_bins: has_integer_bins(feature_name),
} }
} }
}) })
@ -244,6 +250,7 @@ pub fn compute_feature_stats(
p99, p99,
middle_width, middle_width,
num_bins, num_bins,
integer_bins,
.. ..
} => { } => {
let value = data.get_feature(row, fi); let value = data.get_feature(row, fi);
@ -257,12 +264,15 @@ pub fn compute_feature_stats(
} }
*sum += value as f64; *sum += value as f64;
let bin = if value < *p1 { // Snap quantization noise (N decodes to N-epsilon) to the true
// integer so the bin matches the axis label and the mean readout.
let bin_value = if *integer_bins { value.round() } else { value };
let bin = if bin_value < *p1 {
0 0
} else if value >= *p99 { } else if bin_value >= *p99 {
*num_bins - 1 *num_bins - 1
} else if *middle_width > 0.0 { } else if *middle_width > 0.0 {
let middle_bin = ((value - *p1) / *middle_width) as usize; let middle_bin = ((bin_value - *p1) / *middle_width) as usize;
(1 + middle_bin).min(*num_bins - 2) (1 + middle_bin).min(*num_bins - 2)
} else { } else {
*num_bins / 2 *num_bins / 2
@ -570,6 +580,9 @@ pub fn compute_poi_feature_stats(
} else { } else {
0.0 0.0
}; };
// POI count features are u16-quantized integer counts (see the numeric
// path); distance features are fractional and left untouched.
let integer_bins = has_integer_bins(name);
let mut count = 0usize; let mut count = 0usize;
let mut min_value = f32::INFINITY; let mut min_value = f32::INFINITY;
@ -591,12 +604,13 @@ pub fn compute_poi_feature_stats(
} }
sum += value as f64; sum += value as f64;
let bin = if value < p1 { let bin_value = if integer_bins { value.round() } else { value };
let bin = if bin_value < p1 {
0 0
} else if value >= p99 { } else if bin_value >= p99 {
num_bins - 1 num_bins - 1
} else if middle_width > 0.0 { } else if middle_width > 0.0 {
let middle_bin = ((value - p1) / middle_width) as usize; let middle_bin = ((bin_value - p1) / middle_width) as usize;
(1 + middle_bin).min(num_bins - 2) (1 + middle_bin).min(num_bins - 2)
} else { } else {
num_bins / 2 num_bins / 2
@ -732,4 +746,57 @@ mod tests {
let (outcode, sector, out) = area_crime_averages_for(None, &avgs, false, &HashSet::new()); let (outcode, sector, out) = area_crime_averages_for(None, &avgs, false, &HashSet::new());
assert!(outcode.is_none() && sector.is_none() && out.is_empty()); assert!(outcode.is_none() && sector.is_none() && out.is_empty());
} }
#[test]
fn integer_count_bar_aligns_with_label_despite_quant_noise() {
// A genuine catchment count of 2 is u16-quantized over the feature's
// [0, 11] data range, so it decodes to ~1.99995 (just below 2). The
// per-area histogram must snap it to 2 before binning, so the lit bar
// sits under label "2" instead of flooring into the "1" bucket while the
// value readout separately rounds to "2.0".
let name = "Good+ secondary school catchments";
assert!(has_integer_bins(name), "test feature must use integer bins");
let qmin = 0.0f32;
let qrange = 11.0f32;
let dequant_a = qrange / crate::consts::QUANT_SCALE;
// Encode integer 2 exactly as the loader does (round-to-nearest u16).
let raw = ((2.0 - qmin) / qrange * crate::consts::QUANT_SCALE).round() as u16;
let data = PropertyData::numeric_for_tests(vec![raw], 1, vec![dequant_a], vec![qmin]);
let decoded = data.get_feature(0, 0);
assert!(
(1.99..2.0).contains(&decoded),
"expected N-epsilon, got {decoded}"
);
// Global histogram: integer bins over p1=0..p99=7, one unit-wide bin per
// value. 9 bins = [<0], 0,1,2,3,4,5,6, [>=7]; bin index 3 carries label "2".
let feature_stats = vec![FeatureStats {
slider_min: 0.0,
slider_max: 11.0,
histogram: crate::data::Histogram {
min: 0.0,
max: 11.0,
p1: 0.0,
p99: 7.0,
counts: vec![0; 9],
},
}];
let (numeric, _) = compute_feature_stats(
&[0],
&data,
&[name.to_string()],
&FxHashMap::default(),
&feature_stats,
false,
&HashSet::new(),
);
assert_eq!(numeric.len(), 1);
let counts = &numeric[0].histogram.counts;
assert_eq!(counts[3], 1, "value 2 must bin under its own label \"2\"");
assert_eq!(counts[2], 0, "value 2 must not floor into the \"1\" bucket");
// The mean readout is left on the raw decoded value (unchanged by the fix).
assert!(numeric[0].mean < 2.0);
}
} }