All good
This commit is contained in:
parent
6ea544a0f6
commit
6cc7288126
45 changed files with 929 additions and 1043 deletions
|
|
@ -6,6 +6,8 @@ use polars::prelude::*;
|
|||
use serde::Serialize;
|
||||
use tracing::info;
|
||||
|
||||
use crate::consts::{NAN_U16, QUANT_SCALE};
|
||||
use crate::data::{PropertyData, QuantRef};
|
||||
use crate::utils::{normalize_postcode, GridIndex, InternedColumn};
|
||||
|
||||
const GRID_CELL_SIZE: f32 = 0.01;
|
||||
|
|
@ -52,15 +54,22 @@ pub struct ActualListingData {
|
|||
pub listing_status: InternedColumn,
|
||||
pub listing_date_iso: Vec<Option<String>>,
|
||||
pub features: Vec<Vec<String>>,
|
||||
/// Row-major feature matrix aligned with PropertyData::feature_names.
|
||||
///
|
||||
/// Rows start from a best-effort address/postcode join to the historical property
|
||||
/// dataset, then live listing fields such as asking price and property type are
|
||||
/// overlaid where available. This lets the listings endpoint use the same filter
|
||||
/// execution path as the property endpoints.
|
||||
pub filter_feature_data: Vec<u16>,
|
||||
pub grid: GridIndex,
|
||||
}
|
||||
|
||||
impl ActualListingData {
|
||||
pub fn load(parquet_path: &Path) -> Result<Self> {
|
||||
super::run_polars_io(|| Self::load_inner(parquet_path))
|
||||
pub fn load(parquet_path: &Path, property_data: &PropertyData) -> Result<Self> {
|
||||
super::run_polars_io(|| Self::load_inner(parquet_path, Some(property_data)))
|
||||
}
|
||||
|
||||
fn load_inner(parquet_path: &Path) -> Result<Self> {
|
||||
fn load_inner(parquet_path: &Path, property_data: Option<&PropertyData>) -> Result<Self> {
|
||||
info!("Loading actual listings from {:?}", parquet_path);
|
||||
let pl_path = PlRefPath::try_from_path(parquet_path)
|
||||
.context("Failed to normalize actual listings parquet path")?;
|
||||
|
|
@ -99,6 +108,18 @@ impl ActualListingData {
|
|||
let price_qualifier = InternedColumn::build(&opt_to_string(&price_qualifier_raw));
|
||||
let listing_status = InternedColumn::build(&opt_to_string(&listing_status_raw));
|
||||
|
||||
let filter_feature_data = build_filter_feature_data(
|
||||
property_data,
|
||||
&postcode,
|
||||
&address,
|
||||
&property_type_raw,
|
||||
&leasehold_freehold_raw,
|
||||
&rooms_total,
|
||||
&floor_area_sqm,
|
||||
&asking_price,
|
||||
&asking_price_per_sqm,
|
||||
);
|
||||
|
||||
let grid = GridIndex::build(&lat, &lon, GRID_CELL_SIZE);
|
||||
|
||||
info!(rows = row_count, "Actual listings loaded");
|
||||
|
|
@ -122,6 +143,7 @@ impl ActualListingData {
|
|||
listing_status,
|
||||
listing_date_iso,
|
||||
features,
|
||||
filter_feature_data,
|
||||
grid,
|
||||
})
|
||||
}
|
||||
|
|
@ -150,6 +172,201 @@ impl ActualListingData {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_filter_feature_data(
|
||||
property_data: Option<&PropertyData>,
|
||||
postcode: &[String],
|
||||
address: &[Option<String>],
|
||||
property_type: &[Option<String>],
|
||||
leasehold_freehold: &[Option<String>],
|
||||
rooms_total: &[Option<i32>],
|
||||
floor_area_sqm: &[Option<f32>],
|
||||
asking_price: &[Option<i64>],
|
||||
asking_price_per_sqm: &[Option<f32>],
|
||||
) -> Vec<u16> {
|
||||
let Some(property_data) = property_data else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let num_features = property_data.num_features;
|
||||
let mut feature_data = vec![NAN_U16; postcode.len() * num_features];
|
||||
let mut joined_rows = 0usize;
|
||||
|
||||
for (row, postcode_value) in postcode.iter().enumerate() {
|
||||
let Some(address_value) = address[row]
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let query = format!("{address_value} {postcode_value}");
|
||||
let Some(&property_row) = property_data.search_addresses(&query, 1).first() else {
|
||||
continue;
|
||||
};
|
||||
if property_data.postcode(property_row) != postcode_value {
|
||||
continue;
|
||||
}
|
||||
|
||||
let dst = row * num_features;
|
||||
let src = property_row * num_features;
|
||||
feature_data[dst..dst + num_features]
|
||||
.copy_from_slice(&property_data.feature_data[src..src + num_features]);
|
||||
joined_rows += 1;
|
||||
}
|
||||
|
||||
let quant = property_data.quant_ref();
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Total floor area (sqm)",
|
||||
floor_area_sqm.iter().copied(),
|
||||
false,
|
||||
);
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Number of bedrooms & living rooms",
|
||||
rooms_total.iter().map(|value| value.map(|v| v as f32)),
|
||||
false,
|
||||
);
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Estimated current price",
|
||||
asking_price.iter().map(|value| value.map(|v| v as f32)),
|
||||
true,
|
||||
);
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Last known price",
|
||||
asking_price.iter().map(|value| value.map(|v| v as f32)),
|
||||
true,
|
||||
);
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Est. price per sqm",
|
||||
asking_price_per_sqm.iter().copied(),
|
||||
true,
|
||||
);
|
||||
overlay_numeric_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
&quant,
|
||||
"Price per sqm",
|
||||
asking_price_per_sqm.iter().copied(),
|
||||
true,
|
||||
);
|
||||
overlay_enum_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
"Property type",
|
||||
property_type.iter().map(Option::as_deref),
|
||||
false,
|
||||
);
|
||||
overlay_enum_feature(
|
||||
&mut feature_data,
|
||||
property_data,
|
||||
"Leasehold/Freehold",
|
||||
leasehold_freehold.iter().map(Option::as_deref),
|
||||
false,
|
||||
);
|
||||
|
||||
info!(
|
||||
rows = postcode.len(),
|
||||
joined_rows, "Actual listings joined to property feature matrix"
|
||||
);
|
||||
|
||||
feature_data
|
||||
}
|
||||
|
||||
fn feature_index(property_data: &PropertyData, name: &str) -> Option<usize> {
|
||||
property_data
|
||||
.feature_names
|
||||
.iter()
|
||||
.position(|candidate| candidate == name)
|
||||
}
|
||||
|
||||
fn overlay_numeric_feature<I>(
|
||||
feature_data: &mut [u16],
|
||||
property_data: &PropertyData,
|
||||
quant: &QuantRef<'_>,
|
||||
name: &str,
|
||||
values: I,
|
||||
clear_missing: bool,
|
||||
) where
|
||||
I: IntoIterator<Item = Option<f32>>,
|
||||
{
|
||||
let Some(feat_idx) = feature_index(property_data, name) else {
|
||||
return;
|
||||
};
|
||||
if feat_idx >= property_data.num_numeric {
|
||||
return;
|
||||
}
|
||||
|
||||
let num_features = property_data.num_features;
|
||||
for (row, value) in values.into_iter().enumerate() {
|
||||
let dst = row * num_features + feat_idx;
|
||||
match value {
|
||||
Some(value) => feature_data[dst] = encode_numeric_value(quant, feat_idx, value),
|
||||
None if clear_missing => feature_data[dst] = NAN_U16,
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn overlay_enum_feature<'a, I>(
|
||||
feature_data: &mut [u16],
|
||||
property_data: &PropertyData,
|
||||
name: &str,
|
||||
values: I,
|
||||
clear_missing: bool,
|
||||
) where
|
||||
I: IntoIterator<Item = Option<&'a str>>,
|
||||
{
|
||||
let Some(feat_idx) = feature_index(property_data, name) else {
|
||||
return;
|
||||
};
|
||||
let Some(enum_values) = property_data.enum_values.get(&feat_idx) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let num_features = property_data.num_features;
|
||||
for (row, value) in values.into_iter().enumerate() {
|
||||
let dst = row * num_features + feat_idx;
|
||||
let encoded = value
|
||||
.map(str::trim)
|
||||
.filter(|text| !text.is_empty())
|
||||
.and_then(|text| enum_values.iter().position(|candidate| candidate == text))
|
||||
.map(|position| position as u16);
|
||||
match encoded {
|
||||
Some(value) => feature_data[dst] = value,
|
||||
None if clear_missing => feature_data[dst] = NAN_U16,
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_numeric_value(quant: &QuantRef<'_>, feat_idx: usize, value: f32) -> u16 {
|
||||
if !value.is_finite() {
|
||||
return NAN_U16;
|
||||
}
|
||||
let range = quant.quant_range[feat_idx];
|
||||
if range <= 0.0 {
|
||||
return 0;
|
||||
}
|
||||
let normalized = (value - quant.quant_min[feat_idx]) / range;
|
||||
(normalized * QUANT_SCALE).round().clamp(0.0, QUANT_SCALE) as u16
|
||||
}
|
||||
|
||||
fn opt_to_string(values: &[Option<String>]) -> Vec<String> {
|
||||
values
|
||||
.iter()
|
||||
|
|
@ -311,7 +528,7 @@ mod tests {
|
|||
return;
|
||||
};
|
||||
|
||||
let data = ActualListingData::load(&path).expect("listings load");
|
||||
let data = ActualListingData::load_inner(&path, None).expect("listings load");
|
||||
assert!(!data.lat.is_empty());
|
||||
assert_eq!(data.lat.len(), data.lon.len());
|
||||
assert_eq!(data.lat.len(), data.postcode.len());
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue