Fix crime & add actual listings

server-rs/src/data/actual_listings.rs

@@ -0,0 +1,326 @@
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use polars::lazy::frame::LazyFrame;
+use polars::prelude::*;
+use serde::Serialize;
+use tracing::info;
+
+use crate::utils::{normalize_postcode, GridIndex, InternedColumn};
+
+const GRID_CELL_SIZE: f32 = 0.01;
+
+#[derive(Serialize, Clone)]
+pub struct ActualListing {
+    pub lat: f32,
+    pub lon: f32,
+    pub postcode: String,
+    pub address: Option<String>,
+    pub property_type: Option<String>,
+    pub property_sub_type: Option<String>,
+    pub leasehold_freehold: Option<String>,
+    pub price_qualifier: Option<String>,
+    pub bedrooms: Option<i32>,
+    pub bathrooms: Option<i32>,
+    pub rooms_total: Option<i32>,
+    pub floor_area_sqm: Option<f32>,
+    pub asking_price: Option<i64>,
+    pub asking_price_per_sqm: Option<f32>,
+    pub listing_url: String,
+    pub listing_status: Option<String>,
+    pub listing_date_iso: Option<String>,
+    pub features: Vec<String>,
+}
+
+pub struct ActualListingData {
+    pub lat: Vec<f32>,
+    pub lon: Vec<f32>,
+    /// Normalized (uppercase, canonical spacing) postcode per row.
+    pub postcode: Vec<String>,
+    pub address: Vec<Option<String>>,
+    pub property_type: InternedColumn,
+    pub property_sub_type: InternedColumn,
+    pub leasehold_freehold: InternedColumn,
+    pub price_qualifier: InternedColumn,
+    pub bedrooms: Vec<Option<i32>>,
+    pub bathrooms: Vec<Option<i32>>,
+    pub rooms_total: Vec<Option<i32>>,
+    pub floor_area_sqm: Vec<Option<f32>>,
+    pub asking_price: Vec<Option<i64>>,
+    pub asking_price_per_sqm: Vec<Option<f32>>,
+    pub listing_url: Vec<String>,
+    pub listing_status: InternedColumn,
+    pub listing_date_iso: Vec<Option<String>>,
+    pub features: Vec<Vec<String>>,
+    pub grid: GridIndex,
+}
+
+impl ActualListingData {
+    pub fn load(parquet_path: &Path) -> Result<Self> {
+        super::run_polars_io(|| Self::load_inner(parquet_path))
+    }
+
+    fn load_inner(parquet_path: &Path) -> Result<Self> {
+        info!("Loading actual listings from {:?}", parquet_path);
+        let pl_path = PlRefPath::try_from_path(parquet_path)
+            .context("Failed to normalize actual listings parquet path")?;
+        let df = LazyFrame::scan_parquet(pl_path, Default::default())
+            .context("Failed to scan actual listings parquet")?
+            .collect()
+            .context("Failed to read actual listings parquet")?;
+
+        let row_count = df.height();
+        info!(rows = row_count, "Actual listings parquet read");
+
+        let lat = extract_f32(&df, "lat")?;
+        let lon = extract_f32(&df, "lon")?;
+        let postcode_raw = extract_str(&df, "Postcode")?;
+        let address = extract_opt_str(&df, "Address per Property Register")?;
+        let property_type_raw = extract_opt_str(&df, "Property type")?;
+        let property_sub_type_raw = extract_opt_str(&df, "Property sub-type")?;
+        let leasehold_freehold_raw = extract_opt_str(&df, "Leasehold/Freehold")?;
+        let price_qualifier_raw = extract_opt_str(&df, "Price qualifier")?;
+        let bedrooms = extract_opt_i32(&df, "Bedrooms")?;
+        let bathrooms = extract_opt_i32(&df, "Bathrooms")?;
+        let rooms_total = extract_opt_i32(&df, "Number of bedrooms & living rooms")?;
+        let floor_area_sqm = extract_opt_f32(&df, "Total floor area (sqm)")?;
+        let asking_price = extract_opt_i64(&df, "Asking price")?;
+        let asking_price_per_sqm = extract_opt_f32(&df, "Asking price per sqm")?;
+        let listing_url = extract_str(&df, "Listing URL")?;
+        let listing_status_raw = extract_opt_str(&df, "Listing status")?;
+        let listing_date_iso = extract_opt_datetime_iso(&df, "Listing date")?;
+        let features = extract_str_list(&df, "Listing features")?;
+
+        let postcode: Vec<String> = postcode_raw.iter().map(|s| normalize_postcode(s)).collect();
+
+        let property_type = InternedColumn::build(&opt_to_string(&property_type_raw));
+        let property_sub_type = InternedColumn::build(&opt_to_string(&property_sub_type_raw));
+        let leasehold_freehold = InternedColumn::build(&opt_to_string(&leasehold_freehold_raw));
+        let price_qualifier = InternedColumn::build(&opt_to_string(&price_qualifier_raw));
+        let listing_status = InternedColumn::build(&opt_to_string(&listing_status_raw));
+
+        let grid = GridIndex::build(&lat, &lon, GRID_CELL_SIZE);
+
+        info!(rows = row_count, "Actual listings loaded");
+
+        Ok(Self {
+            lat,
+            lon,
+            postcode,
+            address,
+            property_type,
+            property_sub_type,
+            leasehold_freehold,
+            price_qualifier,
+            bedrooms,
+            bathrooms,
+            rooms_total,
+            floor_area_sqm,
+            asking_price,
+            asking_price_per_sqm,
+            listing_url,
+            listing_status,
+            listing_date_iso,
+            features,
+            grid,
+        })
+    }
+
+    pub fn listing_at(&self, row: usize) -> ActualListing {
+        ActualListing {
+            lat: self.lat[row],
+            lon: self.lon[row],
+            postcode: self.postcode[row].clone(),
+            address: self.address[row].clone(),
+            property_type: opt_from_interned(&self.property_type, row),
+            property_sub_type: opt_from_interned(&self.property_sub_type, row),
+            leasehold_freehold: opt_from_interned(&self.leasehold_freehold, row),
+            price_qualifier: opt_from_interned(&self.price_qualifier, row),
+            bedrooms: self.bedrooms[row],
+            bathrooms: self.bathrooms[row],
+            rooms_total: self.rooms_total[row],
+            floor_area_sqm: self.floor_area_sqm[row],
+            asking_price: self.asking_price[row],
+            asking_price_per_sqm: self.asking_price_per_sqm[row],
+            listing_url: self.listing_url[row].clone(),
+            listing_status: opt_from_interned(&self.listing_status, row),
+            listing_date_iso: self.listing_date_iso[row].clone(),
+            features: self.features[row].clone(),
+        }
+    }
+}
+
+fn opt_to_string(values: &[Option<String>]) -> Vec<String> {
+    values
+        .iter()
+        .map(|value| value.clone().unwrap_or_default())
+        .collect()
+}
+
+fn opt_from_interned(column: &InternedColumn, row: usize) -> Option<String> {
+    let value = column.get(row);
+    if value.is_empty() {
+        None
+    } else {
+        Some(value.to_string())
+    }
+}
+
+fn extract_f32(df: &DataFrame, name: &str) -> Result<Vec<f32>> {
+    let cast = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?
+        .cast(&DataType::Float32)
+        .with_context(|| format!("Failed to cast '{name}' to Float32"))?;
+    let column = cast
+        .f32()
+        .with_context(|| format!("Column '{name}' is not Float32"))?;
+    column
+        .into_iter()
+        .enumerate()
+        .map(|(row, value)| value.with_context(|| format!("Column '{name}' has null at row {row}")))
+        .collect()
+}
+
+fn extract_str(df: &DataFrame, name: &str) -> Result<Vec<String>> {
+    let column = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?;
+    let strings = column
+        .str()
+        .with_context(|| format!("Column '{name}' is not a string column"))?;
+    strings
+        .into_iter()
+        .enumerate()
+        .map(|(row, value)| {
+            value
+                .map(ToString::to_string)
+                .with_context(|| format!("Column '{name}' has null at row {row}"))
+        })
+        .collect()
+}
+
+fn extract_opt_str(df: &DataFrame, name: &str) -> Result<Vec<Option<String>>> {
+    let column = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?;
+    let strings = column
+        .str()
+        .with_context(|| format!("Column '{name}' is not a string column"))?;
+    Ok(strings
+        .into_iter()
+        .map(|value| value.and_then(|text| (!text.is_empty()).then(|| text.to_string())))
+        .collect())
+}
+
+fn extract_opt_i32(df: &DataFrame, name: &str) -> Result<Vec<Option<i32>>> {
+    let cast = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?
+        .cast(&DataType::Int32)
+        .with_context(|| format!("Failed to cast '{name}' to Int32"))?;
+    let column = cast
+        .i32()
+        .with_context(|| format!("Column '{name}' is not Int32"))?;
+    Ok(column.into_iter().collect())
+}
+
+fn extract_opt_i64(df: &DataFrame, name: &str) -> Result<Vec<Option<i64>>> {
+    let cast = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?
+        .cast(&DataType::Int64)
+        .with_context(|| format!("Failed to cast '{name}' to Int64"))?;
+    let column = cast
+        .i64()
+        .with_context(|| format!("Column '{name}' is not Int64"))?;
+    Ok(column.into_iter().collect())
+}
+
+fn extract_opt_f32(df: &DataFrame, name: &str) -> Result<Vec<Option<f32>>> {
+    let cast = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?
+        .cast(&DataType::Float32)
+        .with_context(|| format!("Failed to cast '{name}' to Float32"))?;
+    let column = cast
+        .f32()
+        .with_context(|| format!("Column '{name}' is not Float32"))?;
+    Ok(column
+        .into_iter()
+        .map(|value| value.filter(|v| v.is_finite()))
+        .collect())
+}
+
+fn extract_opt_datetime_iso(df: &DataFrame, name: &str) -> Result<Vec<Option<String>>> {
+    let column = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?;
+    let cast = column
+        .cast(&DataType::Datetime(TimeUnit::Microseconds, None))
+        .with_context(|| format!("Failed to cast '{name}' to Datetime(us)"))?;
+    let datetime = cast
+        .datetime()
+        .with_context(|| format!("Column '{name}' is not a Datetime column"))?;
+    Ok(datetime
+        .as_datetime_iter()
+        .map(|value| value.map(|date| date.format("%Y-%m-%dT%H:%M:%SZ").to_string()))
+        .collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::PathBuf;
+
+    fn sample_path() -> Option<PathBuf> {
+        let path = PathBuf::from("../finder/data/online_listings_buy.parquet");
+        path.exists().then_some(path)
+    }
+
+    #[test]
+    fn loads_sample_listings_when_available() {
+        let Some(path) = sample_path() else {
+            eprintln!("sample parquet not present; skipping");
+            return;
+        };
+
+        let data = ActualListingData::load(&path).expect("listings load");
+        assert!(!data.lat.is_empty());
+        assert_eq!(data.lat.len(), data.lon.len());
+        assert_eq!(data.lat.len(), data.postcode.len());
+        assert_eq!(data.lat.len(), data.listing_url.len());
+        assert_eq!(data.lat.len(), data.features.len());
+
+        let any_listing = data.listing_at(0);
+        assert!(any_listing.lat.is_finite());
+        assert!(any_listing.lon.is_finite());
+        assert!(!any_listing.listing_url.is_empty());
+    }
+}
+
+fn extract_str_list(df: &DataFrame, name: &str) -> Result<Vec<Vec<String>>> {
+    let column = df
+        .column(name)
+        .with_context(|| format!("Missing column '{name}'"))?;
+    let list = column
+        .list()
+        .with_context(|| format!("Column '{name}' is not a list column"))?;
+    let mut out = Vec::with_capacity(list.len());
+    for series_opt in list.into_iter() {
+        let entries = match series_opt {
+            Some(series) => {
+                let strings = series.str().with_context(|| {
+                    format!("Column '{name}' list inner is not a string column")
+                })?;
+                strings
+                    .into_iter()
+                    .filter_map(|value| value.map(ToString::to_string))
+                    .collect()
+            }
+            None => Vec::new(),
+        };
+        out.push(entries);
+    }
+    Ok(out)
+}