fine 2
This commit is contained in:
parent
9e4e65fa2a
commit
ca771a7edf
32 changed files with 1467 additions and 109 deletions
|
|
@ -7,7 +7,10 @@ use serde::Serialize;
|
|||
use tracing::info;
|
||||
|
||||
use crate::consts::{NAN_U16, QUANT_SCALE};
|
||||
use crate::data::{PropertyData, QuantRef};
|
||||
use crate::data::{
|
||||
combine_nearest_distances, combined_station_feature_name,
|
||||
combined_station_source_feature_names, PropertyData, QuantRef,
|
||||
};
|
||||
use crate::utils::{normalize_postcode, GridIndex, InternedColumn};
|
||||
|
||||
const GRID_CELL_SIZE: f32 = 0.01;
|
||||
|
|
@ -318,8 +321,17 @@ fn build_poi_filter_feature_data(
|
|||
let mut encoded_columns = 0usize;
|
||||
|
||||
for (metric_idx, name) in poi_metrics.feature_names.iter().enumerate() {
|
||||
let Some(values) = extract_optional_feature_f32(df, name)? else {
|
||||
continue;
|
||||
let values = match extract_optional_feature_f32(df, name)? {
|
||||
Some(values) => values,
|
||||
// Some POI columns (the combined-station distance) are synthesized
|
||||
// in memory by PostcodePoiMetrics and never written to the listings
|
||||
// parquet. Reconstruct them from their source columns so listing POI
|
||||
// filters match the map exactly instead of silently rejecting every
|
||||
// row on an all-NaN column.
|
||||
None => match synthesize_missing_poi_column(df, name, row_count)? {
|
||||
Some(values) => values,
|
||||
None => continue,
|
||||
},
|
||||
};
|
||||
for (row, value) in values.into_iter().enumerate() {
|
||||
let dst = row * num_features + metric_idx;
|
||||
|
|
@ -338,6 +350,45 @@ fn build_poi_filter_feature_data(
|
|||
Ok(feature_data)
|
||||
}
|
||||
|
||||
/// Reconstruct a POI metric column that `PostcodePoiMetrics` synthesizes in
|
||||
/// memory and therefore is absent from the listings parquet. Currently only the
|
||||
/// combined-station distance, derived as the elementwise nearest of its per-mode
|
||||
/// source columns (the same rule the postcode side table uses). Returns `None`
|
||||
/// if `name` is not a synthesized column, or if none of its sources are present.
|
||||
fn synthesize_missing_poi_column(
|
||||
df: &DataFrame,
|
||||
name: &str,
|
||||
row_count: usize,
|
||||
) -> Result<Option<Vec<Option<f32>>>> {
|
||||
if name != combined_station_feature_name() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut sources: Vec<Vec<f32>> = Vec::new();
|
||||
for source_name in combined_station_source_feature_names() {
|
||||
if let Some(values) = extract_optional_feature_f32(df, &source_name)? {
|
||||
sources.push(
|
||||
values
|
||||
.into_iter()
|
||||
.map(|value| value.unwrap_or(f32::NAN))
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
}
|
||||
if sources.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let source_refs: Vec<&[f32]> = sources.iter().map(Vec::as_slice).collect();
|
||||
let combined = combine_nearest_distances(&source_refs, row_count);
|
||||
Ok(Some(
|
||||
combined
|
||||
.into_iter()
|
||||
.map(|value| value.is_finite().then_some(value))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn feature_index(property_data: &PropertyData, name: &str) -> Option<usize> {
|
||||
property_data
|
||||
.feature_names
|
||||
|
|
@ -741,6 +792,51 @@ mod tests {
|
|||
assert!(!any_listing.listing_url.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthesizes_combined_station_column_from_source_columns() {
|
||||
// Mirrors the postcode side table: the combined-station distance is the
|
||||
// finite minimum of the per-mode source columns present in the parquet.
|
||||
let df = df![
|
||||
"Distance to nearest amenity (Rail station) (km)" => [2.0f32, f32::NAN, 5.0],
|
||||
"Distance to nearest amenity (Tube station) (km)" => [1.0f32, f32::NAN, f32::NAN],
|
||||
"Distance to nearest amenity (DLR station) (km)" => [3.0f32, 4.0, f32::NAN],
|
||||
]
|
||||
.unwrap();
|
||||
|
||||
let combined = synthesize_missing_poi_column(&df, &combined_station_feature_name(), 3)
|
||||
.unwrap()
|
||||
.expect("combined-station column should be synthesized");
|
||||
|
||||
assert_eq!(combined[0], Some(1.0)); // min(2, 1, 3)
|
||||
assert_eq!(combined[1], Some(4.0)); // only DLR present
|
||||
assert_eq!(combined[2], Some(5.0)); // only Rail present
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_synthesize_non_combined_or_sourceless_columns() {
|
||||
let df = df![
|
||||
"Distance to nearest amenity (Rail station) (km)" => [1.0f32],
|
||||
]
|
||||
.unwrap();
|
||||
|
||||
// A real per-mode column is read from the parquet, never synthesized.
|
||||
assert!(synthesize_missing_poi_column(
|
||||
&df,
|
||||
"Distance to nearest amenity (Rail station) (km)",
|
||||
1
|
||||
)
|
||||
.unwrap()
|
||||
.is_none());
|
||||
|
||||
// The combined column cannot be built when no source columns exist.
|
||||
let empty = df!["Postcode" => ["AB1 2CD"]].unwrap();
|
||||
assert!(
|
||||
synthesize_missing_poi_column(&empty, &combined_station_feature_name(), 1)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_price_history_from_parquet() {
|
||||
let path = PathBuf::from("src/data/testdata/listings_price_history.parquet");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue