new demo mode & tenure

server-rs/src/data/property/address_search.rs

@@ -251,6 +251,26 @@ pub(super) fn address_search_tokens(text: &str) -> Vec<String> {
     tokens
 }
 
+/// Search tokens for a property row, covering its display address plus an
+/// alternative spelling (`alt`, the EPC form) when distinct — so the property is
+/// findable by either form. The result is deduped: each token appears at most
+/// once, which the inverted-index posting lists rely on (a token must post a
+/// given row exactly once to keep those lists strictly ascending and unique).
+pub(super) fn merged_address_search_tokens(display: &str, alt: Option<&str>) -> Vec<String> {
+    let mut tokens = address_search_tokens(display);
+    if let Some(alt) = alt {
+        let alt = alt.trim();
+        if !alt.is_empty() && alt != display.trim() {
+            for token in address_search_tokens(alt) {
+                if !tokens.contains(&token) {
+                    tokens.push(token);
+                }
+            }
+        }
+    }
+    tokens
+}
+
 fn is_address_search_token(token: &str) -> bool {
     if looks_like_postcode_fragment(token) {
         return false;
@@ -929,6 +949,60 @@ mod tests {
         assert_eq!(parsed.text_groups[1].alternatives, vec!["cour".to_string()]);
     }
 
+    #[test]
+    fn merged_tokens_index_both_address_forms() {
+        // A property whose price-paid form is "10 Park Road" but whose EPC form
+        // names the building ("Kingswood House 10 Park Road") must be findable by
+        // the building name too: the distinctive "kingswood" token comes only
+        // from the EPC form.
+        let display_only = address_search_tokens("10 Park Road");
+        assert!(!display_only.contains(&"kingswood".to_string()));
+
+        let merged =
+            merged_address_search_tokens("10 Park Road", Some("Kingswood House 10 Park Road"));
+        assert!(merged.contains(&"kingswood".to_string()));
+        // Tokens shared by both forms are still present and not duplicated.
+        assert!(merged.contains(&"park".to_string()));
+        assert!(merged.contains(&"road".to_string()));
+        assert!(merged.contains(&"10".to_string()));
+
+        // No token repeats — posting-list uniqueness depends on this.
+        let mut deduped = merged.clone();
+        deduped.sort_unstable();
+        deduped.dedup();
+        assert_eq!(deduped.len(), merged.len());
+    }
+
+    #[test]
+    fn merged_tokens_ignore_redundant_or_missing_alt_form() {
+        let baseline = address_search_tokens("12 Baker Street");
+        // An EPC form identical to the display adds nothing.
+        assert_eq!(
+            merged_address_search_tokens("12 Baker Street", Some("12 Baker Street")),
+            baseline
+        );
+        // A blank EPC form (whitespace) and an absent one are both no-ops.
+        assert_eq!(
+            merged_address_search_tokens("12 Baker Street", Some("  ")),
+            baseline
+        );
+        assert_eq!(
+            merged_address_search_tokens("12 Baker Street", None),
+            baseline
+        );
+    }
+
+    #[test]
+    fn merged_tokens_handle_epc_only_display() {
+        // EPC-only dwellings have no price-paid form, so the EPC address IS the
+        // display; passing it as both arguments must not duplicate its tokens.
+        let epc = "Flat 4 Rosewood Court";
+        assert_eq!(
+            merged_address_search_tokens(epc, Some(epc)),
+            address_search_tokens(epc)
+        );
+    }
+
     #[test]
     fn address_search_tokens_keep_actual_address_terms_for_scoring() {
         let tokens = address_search_tokens("Flat 2, 10 Downing Cour");

server-rs/src/data/property/loading.rs

@@ -14,11 +14,11 @@ use crate::consts::{NAN_U16, QUANT_SCALE};
 use crate::features::{self, Bounds};
 
 use super::address_search::{
-    address_search_tokens, build_address_prefix_index, is_address_candidate_token,
+    build_address_prefix_index, is_address_candidate_token, merged_address_search_tokens,
 };
 use super::poi_metrics::{PostcodePoiMetrics, NO_POI_METRIC_ROW};
 use super::stats::{column_to_f32_vec, compute_feature_stats, FeatureStats, Histogram};
-use super::{HistoricalPrice, PropertyData, RenovationEvent};
+use super::{HistoricalPrice, PropertyData, RenovationEvent, TenureEvent};
 
 const MISSING_COORDINATE_SAMPLE_LIMIT: usize = 10;
 const COUNTRY_COLUMN_CANDIDATES: &[&str] = &[
@@ -330,6 +330,10 @@ impl PropertyData {
         if has_historical_prices {
             select_exprs.push(col("historical_prices"));
         }
+        let has_tenure_history = schema.get("tenure_history").is_some();
+        if has_tenure_history {
+            select_exprs.push(col("tenure_history"));
+        }
         let df = combined_lf
             .filter(col("lat").is_not_null().and(col("lon").is_not_null()))
             .select(select_exprs)
@@ -439,7 +443,6 @@ impl PropertyData {
                 .collect()
         };
 
-        let address_raw = extract_string_col(&df, "Address per Property Register")?;
         let postcode_raw = extract_string_col(&df, "Postcode")?;
 
         // Extract optional string columns
@@ -468,6 +471,13 @@ impl PropertyData {
             };
         let property_sub_type_raw = extract_optional_string_col(&df, "Property sub-type")?;
         let price_qualifier_raw = extract_optional_string_col(&df, "Price qualifier")?;
+        // Display + search addresses. The price-paid form ("Address per Property
+        // Register") is the primary display address; the EPC form ("Address per
+        // EPC") is an alternative spelling indexed alongside it so a property is
+        // findable by either. For EPC-only dwellings (no Land Registry sale) the
+        // price-paid form is null, so the EPC form becomes the display address.
+        let pp_address_raw = extract_optional_string_col(&df, "Address per Property Register")?;
+        let epc_address_raw = extract_optional_string_col(&df, "Address per EPC")?;
 
         tracing::info!("Building enum features");
         // enum_col_major: Vec<(values_list, encoded_as_f32)>
@@ -675,6 +685,57 @@ impl PropertyData {
                 FxHashMap::default()
             };
 
+        // Extract tenure_history: List<Struct{year: i32, status: str}>
+        let mut tenure_raw: FxHashMap<u32, Vec<TenureEvent>> = if has_tenure_history {
+            tracing::info!("Extracting tenure history");
+            let tenure_col = df
+                .column("tenure_history")
+                .context("Missing tenure_history column")?;
+            let list_ca = tenure_col
+                .list()
+                .context("tenure_history is not a list column")?;
+
+            let mut history: FxHashMap<u32, Vec<TenureEvent>> = FxHashMap::default();
+            for old_row in 0..row_count {
+                if let Some(inner) = list_ca.get_as_series(old_row) {
+                    if inner.is_empty() {
+                        continue;
+                    }
+                    let structs = inner
+                        .struct_()
+                        .context("tenure_history inner is not a struct")?;
+                    let years = structs
+                        .field_by_name("year")
+                        .context("Missing 'year' field in tenure_history struct")?;
+                    let statuses = structs
+                        .field_by_name("status")
+                        .context("Missing 'status' field in tenure_history struct")?;
+
+                    let mut row_events = Vec::new();
+                    for idx in 0..inner.len() {
+                        let year = years.get(idx).context("Failed to get year value")?;
+                        let status = statuses.get(idx).context("Failed to get status value")?;
+                        if let (AnyValue::Int32(yr), AnyValue::String(st)) = (&year, &status) {
+                            row_events.push(TenureEvent {
+                                year: *yr,
+                                status: st.to_string(),
+                            });
+                        }
+                    }
+                    if !row_events.is_empty() {
+                        history.insert(old_row as u32, row_events);
+                    }
+                }
+            }
+            tracing::info!(
+                properties_with_tenure_changes = history.len(),
+                "Tenure history extracted"
+            );
+            history
+        } else {
+            FxHashMap::default()
+        };
+
         // Free the projected joined frame before building the row-major matrix.
         drop(df);
 
@@ -712,9 +773,21 @@ impl PropertyData {
             })
             .context("Required numeric column 'Last known price' not configured")?;
 
-        // Build contiguous address buffer and address search index (permuted)
+        // Build contiguous address buffer and address search index (permuted).
+        // Each row's posting lists cover BOTH address spellings we hold — the
+        // price-paid form and the EPC form — so a property is findable by either;
+        // the display address prefers the price-paid form, falling back to the EPC
+        // form for never-sold (EPC-only) dwellings whose price-paid form is null.
         tracing::info!("Building interned strings");
-        let total_addr_bytes: usize = address_raw.iter().map(|text| text.len()).sum();
+        // Display address for a row, preferring the price-paid form and falling
+        // back to the EPC form (so EPC-only dwellings still have a display string).
+        let coalesced_display = |old_row: usize| -> &str {
+            match pp_address_raw[old_row].as_deref() {
+                Some(pp) if !pp.is_empty() => pp,
+                _ => epc_address_raw[old_row].as_deref().unwrap_or(""),
+            }
+        };
+        let total_addr_bytes: usize = (0..row_count).map(|row| coalesced_display(row).len()).sum();
         let mut address_buffer = String::with_capacity(total_addr_bytes);
         let mut address_offsets = Vec::with_capacity(row_count);
         let mut address_lengths = Vec::with_capacity(row_count);
@@ -724,14 +797,19 @@ impl PropertyData {
         let mut address_search_token_offsets = Vec::with_capacity(row_count);
         let mut address_search_token_lengths = Vec::with_capacity(row_count);
         for (new_row, &perm_index) in perm.iter().enumerate() {
-            let addr = &address_raw[perm_index as usize];
+            let old_row = perm_index as usize;
+            let display = coalesced_display(old_row);
             let offset = address_buffer.len() as u32;
-            let length = addr.len().min(u16::MAX as usize) as u16;
+            let length = display.len().min(u16::MAX as usize) as u16;
             address_offsets.push(offset);
             address_lengths.push(length);
-            address_buffer.push_str(&addr[..length as usize]);
+            address_buffer.push_str(&display[..length as usize]);
 
-            let search_tokens = address_search_tokens(addr);
+            // Tokens cover the display address plus the EPC-form spelling when
+            // distinct, so the property is findable by either form (deduped so
+            // each token posts this row exactly once).
+            let search_tokens =
+                merged_address_search_tokens(display, epc_address_raw[old_row].as_deref());
             let token_offset = address_search_token_keys.len() as u32;
             let token_length = search_tokens.len().min(u16::MAX as usize) as u16;
             address_search_token_offsets.push(token_offset);
@@ -834,6 +912,18 @@ impl PropertyData {
             map
         };
 
+        // Re-key tenure_history by permuted row index
+        let tenure_history: FxHashMap<u32, Vec<TenureEvent>> = {
+            let mut map =
+                FxHashMap::with_capacity_and_hasher(tenure_raw.len(), Default::default());
+            for (new_row, &old_row) in perm.iter().enumerate() {
+                if let Some(events) = tenure_raw.remove(&old_row) {
+                    map.insert(new_row as u32, events);
+                }
+            }
+            map
+        };
+
         // Permute optional string columns into sparse HashMaps
         let property_sub_type: FxHashMap<u32, String> = {
             let mut map = FxHashMap::default();
@@ -968,6 +1058,7 @@ impl PropertyData {
             approx_build_date_bits,
             renovation_history,
             historical_prices,
+            tenure_history,
             property_sub_type,
             price_qualifier,
         })

server-rs/src/data/property/mod.rs

@@ -40,6 +40,16 @@ pub struct HistoricalPrice {
     pub price: i64,
 }
 
+/// A point on the property's tenure timeline: the year a certificate first
+/// recorded a new occupancy `status` ("Owner-occupied" / "Rented (private)" /
+/// "Rented (social)"). Derived per-EPC and reduced to change points by the
+/// pipeline, so this surfaces *when* a home was let out vs lived in.
+#[derive(Serialize, Clone)]
+pub struct TenureEvent {
+    pub year: i32,
+    pub status: String,
+}
+
 pub struct PropertyData {
     pub lat: Vec<f32>,
     pub lon: Vec<f32>,
@@ -100,6 +110,9 @@ pub struct PropertyData {
     /// Per-row historical sale transactions (Land Registry price-paid).
     /// Keyed by (permuted) row index. Only rows with prices are present.
     historical_prices: FxHashMap<u32, Vec<HistoricalPrice>>,
+    /// Per-row tenure timeline (owner-occupied <-> rented change points from
+    /// EPC). Keyed by (permuted) row index. Only rows with changes are present.
+    tenure_history: FxHashMap<u32, Vec<TenureEvent>>,
     property_sub_type: FxHashMap<u32, String>,
     price_qualifier: FxHashMap<u32, String>,
 }
@@ -153,6 +166,14 @@ impl PropertyData {
             .unwrap_or(&[])
     }
 
+    /// Get tenure timeline events for a given row (empty slice if none).
+    pub fn tenure_history(&self, row: usize) -> &[TenureEvent] {
+        self.tenure_history
+            .get(&(row as u32))
+            .map(|v| v.as_slice())
+            .unwrap_or(&[])
+    }
+
     /// Get property sub-type for a given row.
     pub fn property_sub_type(&self, row: usize) -> Option<&str> {
         self.property_sub_type
@@ -231,6 +252,7 @@ impl PropertyData {
             approx_build_date_bits: Vec::new(),
             renovation_history: FxHashMap::default(),
             historical_prices: FxHashMap::default(),
+            tenure_history: FxHashMap::default(),
             property_sub_type: FxHashMap::default(),
             price_qualifier: FxHashMap::default(),
         }