fix bin silliness
This commit is contained in:
parent
d1faad314b
commit
d78a301028
1 changed files with 73 additions and 6 deletions
|
|
@ -8,6 +8,7 @@ use crate::consts::{AREA_PRICE_HISTORY_POINTS_LIMIT, PRICE_HISTORY_POINTS_LIMIT}
|
|||
use crate::data::area_crime_averages::AreaCrimeAverages;
|
||||
use crate::data::crime_by_year::CrimeByYearData;
|
||||
use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
|
||||
use crate::features::has_integer_bins;
|
||||
use crate::utils::{postcode_outcode, postcode_sector};
|
||||
|
||||
use super::hexagon_stats::{
|
||||
|
|
@ -146,6 +147,10 @@ enum FeatureAccum {
|
|||
num_bins: usize,
|
||||
global_min: f32,
|
||||
global_max: f32,
|
||||
/// Integer-count feature: values are u16-quantized so a genuine integer N
|
||||
/// can decode to N-epsilon; round to the nearest integer before binning so
|
||||
/// the lit bar lands under its own axis label instead of one bucket left.
|
||||
integer_bins: bool,
|
||||
},
|
||||
/// Enum: count occurrences per variant index.
|
||||
Enum { value_counts: Vec<u64> },
|
||||
|
|
@ -202,6 +207,7 @@ pub fn compute_feature_stats(
|
|||
num_bins,
|
||||
global_min: global_hist.min,
|
||||
global_max: global_hist.max,
|
||||
integer_bins: has_integer_bins(feature_name),
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
@ -244,6 +250,7 @@ pub fn compute_feature_stats(
|
|||
p99,
|
||||
middle_width,
|
||||
num_bins,
|
||||
integer_bins,
|
||||
..
|
||||
} => {
|
||||
let value = data.get_feature(row, fi);
|
||||
|
|
@ -257,12 +264,15 @@ pub fn compute_feature_stats(
|
|||
}
|
||||
*sum += value as f64;
|
||||
|
||||
let bin = if value < *p1 {
|
||||
// Snap quantization noise (N decodes to N-epsilon) to the true
|
||||
// integer so the bin matches the axis label and the mean readout.
|
||||
let bin_value = if *integer_bins { value.round() } else { value };
|
||||
let bin = if bin_value < *p1 {
|
||||
0
|
||||
} else if value >= *p99 {
|
||||
} else if bin_value >= *p99 {
|
||||
*num_bins - 1
|
||||
} else if *middle_width > 0.0 {
|
||||
let middle_bin = ((value - *p1) / *middle_width) as usize;
|
||||
let middle_bin = ((bin_value - *p1) / *middle_width) as usize;
|
||||
(1 + middle_bin).min(*num_bins - 2)
|
||||
} else {
|
||||
*num_bins / 2
|
||||
|
|
@ -570,6 +580,9 @@ pub fn compute_poi_feature_stats(
|
|||
} else {
|
||||
0.0
|
||||
};
|
||||
// POI count features are u16-quantized integer counts (see the numeric
|
||||
// path); distance features are fractional and left untouched.
|
||||
let integer_bins = has_integer_bins(name);
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f32::INFINITY;
|
||||
|
|
@ -591,12 +604,13 @@ pub fn compute_poi_feature_stats(
|
|||
}
|
||||
sum += value as f64;
|
||||
|
||||
let bin = if value < p1 {
|
||||
let bin_value = if integer_bins { value.round() } else { value };
|
||||
let bin = if bin_value < p1 {
|
||||
0
|
||||
} else if value >= p99 {
|
||||
} else if bin_value >= p99 {
|
||||
num_bins - 1
|
||||
} else if middle_width > 0.0 {
|
||||
let middle_bin = ((value - p1) / middle_width) as usize;
|
||||
let middle_bin = ((bin_value - p1) / middle_width) as usize;
|
||||
(1 + middle_bin).min(num_bins - 2)
|
||||
} else {
|
||||
num_bins / 2
|
||||
|
|
@ -732,4 +746,57 @@ mod tests {
|
|||
let (outcode, sector, out) = area_crime_averages_for(None, &avgs, false, &HashSet::new());
|
||||
assert!(outcode.is_none() && sector.is_none() && out.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integer_count_bar_aligns_with_label_despite_quant_noise() {
|
||||
// A genuine catchment count of 2 is u16-quantized over the feature's
|
||||
// [0, 11] data range, so it decodes to ~1.99995 (just below 2). The
|
||||
// per-area histogram must snap it to 2 before binning, so the lit bar
|
||||
// sits under label "2" instead of flooring into the "1" bucket while the
|
||||
// value readout separately rounds to "2.0".
|
||||
let name = "Good+ secondary school catchments";
|
||||
assert!(has_integer_bins(name), "test feature must use integer bins");
|
||||
|
||||
let qmin = 0.0f32;
|
||||
let qrange = 11.0f32;
|
||||
let dequant_a = qrange / crate::consts::QUANT_SCALE;
|
||||
// Encode integer 2 exactly as the loader does (round-to-nearest u16).
|
||||
let raw = ((2.0 - qmin) / qrange * crate::consts::QUANT_SCALE).round() as u16;
|
||||
let data = PropertyData::numeric_for_tests(vec![raw], 1, vec![dequant_a], vec![qmin]);
|
||||
let decoded = data.get_feature(0, 0);
|
||||
assert!(
|
||||
(1.99..2.0).contains(&decoded),
|
||||
"expected N-epsilon, got {decoded}"
|
||||
);
|
||||
|
||||
// Global histogram: integer bins over p1=0..p99=7, one unit-wide bin per
|
||||
// value. 9 bins = [<0], 0,1,2,3,4,5,6, [>=7]; bin index 3 carries label "2".
|
||||
let feature_stats = vec![FeatureStats {
|
||||
slider_min: 0.0,
|
||||
slider_max: 11.0,
|
||||
histogram: crate::data::Histogram {
|
||||
min: 0.0,
|
||||
max: 11.0,
|
||||
p1: 0.0,
|
||||
p99: 7.0,
|
||||
counts: vec![0; 9],
|
||||
},
|
||||
}];
|
||||
|
||||
let (numeric, _) = compute_feature_stats(
|
||||
&[0],
|
||||
&data,
|
||||
&[name.to_string()],
|
||||
&FxHashMap::default(),
|
||||
&feature_stats,
|
||||
false,
|
||||
&HashSet::new(),
|
||||
);
|
||||
assert_eq!(numeric.len(), 1);
|
||||
let counts = &numeric[0].histogram.counts;
|
||||
assert_eq!(counts[3], 1, "value 2 must bin under its own label \"2\"");
|
||||
assert_eq!(counts[2], 0, "value 2 must not floor into the \"1\" bucket");
|
||||
// The mean readout is left on the raw decoded value (unchanged by the fix).
|
||||
assert!(numeric[0].mean < 2.0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue