idk

server-rs/src/data/places.rs

@@ -4,10 +4,16 @@ use anyhow::Context;
 use polars::frame::DataFrame;
 use polars::lazy::frame::LazyFrame;
 use polars::prelude::*;
+use rustc_hash::FxHashMap;
 use tracing::info;
 
 use crate::utils::InternedColumn;
 
+/// Upper bound on place rows scored per query (candidate sets are normally far smaller).
+const PLACE_CANDIDATE_LIMIT: usize = 50_000;
+const PLACE_PREFIX_MIN_LEN: usize = 2;
+const PLACE_PREFIX_MAX_LEN: usize = 6;
+
 pub struct PlaceData {
     pub name: Vec<String>,
     pub name_lower: Vec<String>,
@@ -19,6 +25,13 @@ pub struct PlaceData {
     pub lon: Vec<f32>,
     pub city: Vec<Option<String>>,
     pub travel_destination: Vec<bool>,
+    /// Inverted index from an alias token to the (ascending) place rows containing it. Lets place
+    /// search gather candidates instead of scanning all ~1M+ rows per keystroke.
+    token_index: FxHashMap<String, Vec<u32>>,
+    /// Prefix → indexed tokens, for matching a partially-typed final word.
+    token_prefix_index: FxHashMap<String, Vec<String>>,
+    /// Trigram → fuzzy-eligible rows (settlements/stations only), for bounded typo matching.
+    fuzzy_trigram_index: FxHashMap<u32, Vec<u32>>,
 }
 
 #[derive(Clone, Copy)]
@@ -168,6 +181,148 @@ pub fn normalize_search_text(text: &str) -> String {
     result
 }
 
+/// Tokens across all of a place's search aliases (split on word and alias separators),
+/// for token-AND matching where every query word must prefix-match some place token.
+pub fn place_alias_tokens(search_text: &str) -> impl Iterator<Item = &str> {
+    search_text
+        .split([' ', '|'])
+        .filter(|token| !token.is_empty())
+}
+
+fn trigram_hash(first: char, second: char, third: char) -> u32 {
+    let mut hash = 2_166_136_261u32;
+    for ch in [first, second, third] {
+        hash = (hash ^ (ch as u32)).wrapping_mul(16_777_619);
+    }
+    hash
+}
+
+/// Sorted, de-duplicated padded character trigrams of `text`, for Jaccard fuzzy matching.
+pub fn compute_trigrams(text: &str) -> Vec<u32> {
+    let norm = normalize_search_text(text);
+    if norm.is_empty() {
+        return Vec::new();
+    }
+    let chars: Vec<char> = [' ', ' ']
+        .into_iter()
+        .chain(norm.chars())
+        .chain(std::iter::once(' '))
+        .collect();
+    let mut grams: Vec<u32> = chars
+        .windows(3)
+        .map(|window| trigram_hash(window[0], window[1], window[2]))
+        .collect();
+    grams.sort_unstable();
+    grams.dedup();
+    grams
+}
+
+/// Intersect two ascending-sorted row-id slices.
+fn intersect_sorted(left: &[u32], right: &[u32]) -> Vec<u32> {
+    let mut out = Vec::new();
+    let (mut i, mut j) = (0, 0);
+    while i < left.len() && j < right.len() {
+        match left[i].cmp(&right[j]) {
+            std::cmp::Ordering::Less => i += 1,
+            std::cmp::Ordering::Greater => j += 1,
+            std::cmp::Ordering::Equal => {
+                out.push(left[i]);
+                i += 1;
+                j += 1;
+            }
+        }
+    }
+    out
+}
+
+/// Union two ascending-sorted row-id slices (deduplicated, stays sorted).
+fn union_sorted(left: &[u32], right: &[u32]) -> Vec<u32> {
+    let mut out = Vec::with_capacity(left.len() + right.len());
+    let (mut i, mut j) = (0, 0);
+    while i < left.len() && j < right.len() {
+        match left[i].cmp(&right[j]) {
+            std::cmp::Ordering::Less => {
+                out.push(left[i]);
+                i += 1;
+            }
+            std::cmp::Ordering::Greater => {
+                out.push(right[j]);
+                j += 1;
+            }
+            std::cmp::Ordering::Equal => {
+                out.push(left[i]);
+                i += 1;
+                j += 1;
+            }
+        }
+    }
+    out.extend_from_slice(&left[i..]);
+    out.extend_from_slice(&right[j..]);
+    out
+}
+
+/// Distinct indexable tokens (len ≥ 2) across all of a place's search aliases. ASCII because
+/// `normalize_search_text` already dropped non-alphanumerics, so prefix byte-slicing is safe.
+fn place_index_tokens(search_text: &str) -> Vec<String> {
+    let mut tokens: Vec<String> = place_alias_tokens(search_text)
+        .filter(|token| token.len() >= 2)
+        .map(ToString::to_string)
+        .collect();
+    tokens.sort_unstable();
+    tokens.dedup();
+    tokens
+}
+
+fn build_place_prefix_index(
+    token_index: &FxHashMap<String, Vec<u32>>,
+) -> FxHashMap<String, Vec<String>> {
+    let mut prefix_index: FxHashMap<String, Vec<String>> = FxHashMap::default();
+    for token in token_index.keys() {
+        let max_len = token.len().min(PLACE_PREFIX_MAX_LEN);
+        for len in PLACE_PREFIX_MIN_LEN..=max_len {
+            prefix_index
+                .entry(token[..len].to_string())
+                .or_default()
+                .push(token.clone());
+        }
+    }
+    for tokens in prefix_index.values_mut() {
+        tokens.sort_unstable();
+        tokens.dedup();
+    }
+    prefix_index
+}
+
+/// Whether a place type participates in fuzzy (typo) matching. Settlements/stations/universities
+/// do; the ~1M streets and POIs do not (people rarely misspell a road and it keeps fuzzy bounded).
+fn is_fuzzy_eligible_type(place_type: &str) -> bool {
+    !matches!(
+        place_type,
+        "street" | "park" | "attraction" | "hospital" | "retail"
+    )
+}
+
+/// Jaccard similarity between two sorted trigram sets (0.0–1.0).
+pub fn trigram_similarity(left: &[u32], right: &[u32]) -> f32 {
+    if left.is_empty() || right.is_empty() {
+        return 0.0;
+    }
+    let (mut i, mut j, mut intersection) = (0, 0, 0usize);
+    while i < left.len() && j < right.len() {
+        match left[i].cmp(&right[j]) {
+            std::cmp::Ordering::Less => i += 1,
+            std::cmp::Ordering::Greater => j += 1,
+            std::cmp::Ordering::Equal => {
+                intersection += 1;
+                i += 1;
+                j += 1;
+            }
+        }
+    }
+    let union = left.len() + right.len() - intersection;
+    intersection as f32 / union as f32
+}
+
 fn replace_token(text: &str, from: &str, to: &str) -> Option<String> {
     let mut changed = false;
     let replaced: Vec<&str> = text
@@ -191,15 +346,31 @@ fn push_alias(aliases: &mut Vec<String>, alias: String) {
     }
 }
 
+/// Bidirectional token abbreviations expanded into search aliases so a query typed either
+/// way matches (e.g. "gt missenden" ↔ "Great Missenden", "mt" ↔ "Mount").
+const PLACE_TOKEN_ALIASES: &[(&str, &str)] = &[
+    ("st", "saint"),
+    ("saint", "st"),
+    ("mt", "mount"),
+    ("mount", "mt"),
+    ("gt", "great"),
+    ("great", "gt"),
+    ("lt", "little"),
+    ("little", "lt"),
+    ("upr", "upper"),
+    ("upper", "upr"),
+    ("lwr", "lower"),
+    ("lower", "lwr"),
+];
+
 fn build_search_text(name: &str, place_type: &str) -> String {
     let primary = normalize_search_text(name);
     let mut aliases = vec![primary.clone()];
 
-    if let Some(alias) = replace_token(&primary, "st", "saint") {
-        push_alias(&mut aliases, alias);
-    }
-    if let Some(alias) = replace_token(&primary, "saint", "st") {
-        push_alias(&mut aliases, alias);
+    for (from, to) in PLACE_TOKEN_ALIASES {
+        if let Some(alias) = replace_token(&primary, from, to) {
+            push_alias(&mut aliases, alias);
+        }
     }
 
     if place_type == "station" {
@@ -391,6 +562,26 @@ impl PlaceData {
             fallback_city
         };
 
+        // Build the place search index: an inverted token index over all rows (so the per-query
+        // cost scales with matched candidates, not the ~1M-row corpus), plus a trigram index over
+        // only fuzzy-eligible rows for bounded typo matching.
+        let mut token_index: FxHashMap<String, Vec<u32>> = FxHashMap::default();
+        let mut fuzzy_trigram_index: FxHashMap<u32, Vec<u32>> = FxHashMap::default();
+        for idx in 0..row_count {
+            for token in place_index_tokens(&name_search[idx]) {
+                token_index.entry(token).or_default().push(idx as u32);
+            }
+            if is_fuzzy_eligible_type(&place_type_raw[idx]) {
+                for trigram in compute_trigrams(&name[idx]) {
+                    fuzzy_trigram_index
+                        .entry(trigram)
+                        .or_default()
+                        .push(idx as u32);
+                }
+            }
+        }
+        let token_prefix_index = build_place_prefix_index(&token_index);
+
         let with_pop = population.iter().filter(|&&pop| pop > 0).count();
         let with_city = city.iter().filter(|c| c.is_some()).count();
         info!(
@@ -398,6 +589,8 @@ impl PlaceData {
             types = place_type.values.len(),
             with_population = with_pop,
             with_city = with_city,
+            tokens = token_index.len(),
+            fuzzy_trigrams = fuzzy_trigram_index.len(),
             "Place data loaded"
         );
 
@@ -412,14 +605,261 @@ impl PlaceData {
             lon,
             city,
             travel_destination,
+            token_index,
+            token_prefix_index,
+            fuzzy_trigram_index,
         })
     }
+
+    /// Candidate place rows for the query content tokens: intersect the posting lists of words
+    /// typed in full; if none matched an indexed token exactly, seed from the smallest
+    /// prefix-expanded list (so a partially-typed final word still works). Bounded by
+    /// `PLACE_CANDIDATE_LIMIT`.
+    pub fn place_candidate_rows(&self, tokens: &[&str]) -> Vec<u32> {
+        let mut exact: Vec<&[u32]> = tokens
+            .iter()
+            .filter_map(|token| self.token_index.get(*token).map(Vec::as_slice))
+            .collect();
+
+        let mut rows = if exact.is_empty() {
+            self.place_prefix_seed(tokens)
+        } else {
+            exact.sort_by_key(|posting| posting.len());
+            let mut acc = exact[0].to_vec();
+            for posting in &exact[1..] {
+                if acc.is_empty() {
+                    break;
+                }
+                acc = intersect_sorted(&acc, posting);
+            }
+            acc
+        };
+        rows.truncate(PLACE_CANDIDATE_LIMIT);
+        rows
+    }
+
+    fn place_prefix_seed(&self, tokens: &[&str]) -> Vec<u32> {
+        let mut best: Option<Vec<u32>> = None;
+        for token in tokens {
+            if token.len() < PLACE_PREFIX_MIN_LEN {
+                continue;
+            }
+            let key = &token[..token.len().min(PLACE_PREFIX_MAX_LEN)];
+            let Some(indexed) = self.token_prefix_index.get(key) else {
+                continue;
+            };
+            let mut union: Vec<u32> = Vec::new();
+            for indexed_token in indexed {
+                if !indexed_token.starts_with(token) {
+                    continue;
+                }
+                if let Some(rows) = self.token_index.get(indexed_token) {
+                    union = if union.is_empty() {
+                        rows.clone()
+                    } else {
+                        union_sorted(&union, rows)
+                    };
+                }
+            }
+            if !union.is_empty()
+                && best
+                    .as_ref()
+                    .is_none_or(|current| union.len() < current.len())
+            {
+                best = Some(union);
+            }
+        }
+        best.unwrap_or_default()
+    }
+
+    /// Fuzzy-eligible rows sharing enough trigrams with the query to be worth Jaccard scoring.
+    /// Bounded by the (small) fuzzy trigram index rather than scanning every place.
+    pub fn fuzzy_candidate_rows(&self, query_trigrams: &[u32]) -> Vec<u32> {
+        if query_trigrams.is_empty() {
+            return Vec::new();
+        }
+        let mut counts: FxHashMap<u32, u16> = FxHashMap::default();
+        for trigram in query_trigrams {
+            if let Some(rows) = self.fuzzy_trigram_index.get(trigram) {
+                for &row in rows {
+                    *counts.entry(row).or_default() += 1;
+                }
+            }
+        }
+        let min_shared = (((query_trigrams.len() as f32) * 0.4).ceil() as u16).max(1);
+        counts
+            .into_iter()
+            .filter_map(|(row, shared)| (shared >= min_shared).then_some(row))
+            .collect()
+    }
+}
+
+#[cfg(test)]
+impl PlaceData {
+    /// Build a minimal PlaceData from (name, place_type) pairs for index tests.
+    fn from_names<S: AsRef<str>>(rows: &[(S, S)]) -> Self {
+        let name: Vec<String> = rows.iter().map(|(nm, _)| nm.as_ref().to_string()).collect();
+        let place_type_raw: Vec<String> =
+            rows.iter().map(|(_, pt)| pt.as_ref().to_string()).collect();
+        let name_lower: Vec<String> = name.iter().map(|nm| nm.to_lowercase()).collect();
+        let name_search: Vec<String> = name
+            .iter()
+            .zip(&place_type_raw)
+            .map(|(nm, pt)| build_search_text(nm, pt))
+            .collect();
+        let mut token_index: FxHashMap<String, Vec<u32>> = FxHashMap::default();
+        let mut fuzzy_trigram_index: FxHashMap<u32, Vec<u32>> = FxHashMap::default();
+        for idx in 0..name.len() {
+            for token in place_index_tokens(&name_search[idx]) {
+                token_index.entry(token).or_default().push(idx as u32);
+            }
+            if is_fuzzy_eligible_type(&place_type_raw[idx]) {
+                for trigram in compute_trigrams(&name[idx]) {
+                    fuzzy_trigram_index
+                        .entry(trigram)
+                        .or_default()
+                        .push(idx as u32);
+                }
+            }
+        }
+        let token_prefix_index = build_place_prefix_index(&token_index);
+        let len = name.len();
+        PlaceData {
+            name,
+            name_lower,
+            name_search,
+            place_type: InternedColumn::build(&place_type_raw),
+            type_rank: place_type_raw.iter().map(|pt| type_rank(pt)).collect(),
+            population: vec![0; len],
+            lat: vec![0.0; len],
+            lon: vec![0.0; len],
+            city: vec![None; len],
+            travel_destination: vec![false; len],
+            token_index,
+            token_prefix_index,
+            fuzzy_trigram_index,
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn place_index_tokens_dedup_and_min_length() {
+        // "a" is too short; aliases split on " | ".
+        assert_eq!(
+            place_index_tokens("st albans | saint albans"),
+            vec!["albans".to_string(), "saint".to_string(), "st".to_string()]
+        );
+    }
+
+    #[test]
+    fn place_candidate_rows_intersect_and_prefix_seed() {
+        let pd = PlaceData::from_names(&[
+            ("Camden", "suburb"),
+            ("Camden Town", "suburb"),
+            ("Camden Market", "attraction"),
+            ("Manchester", "city"),
+            ("Manchester Piccadilly", "station"),
+        ]);
+
+        // Full word → posting list (Camden, Camden Town, Camden Market).
+        let camden = pd.place_candidate_rows(&["camden"]);
+        assert_eq!(camden, vec![0, 1, 2]);
+
+        // Two full words intersect to rows containing BOTH (Camden Town only).
+        let camden_town = pd.place_candidate_rows(&["camden", "town"]);
+        assert_eq!(camden_town, vec![1]);
+
+        // A partially-typed final word with no exact token seeds from the prefix index.
+        let piccad = pd.place_candidate_rows(&["piccad"]);
+        assert_eq!(piccad, vec![4]);
+
+        // No match → empty.
+        assert!(pd.place_candidate_rows(&["zzzz"]).is_empty());
+    }
+
+    // Run with: cargo test --release bench_place_search -- --ignored --nocapture
+    #[test]
+    #[ignore]
+    fn bench_place_search_at_one_million_rows() {
+        let roads = [
+            "High Street",
+            "Station Road",
+            "Church Lane",
+            "Victoria Road",
+            "Mill Lane",
+            "Park Avenue",
+            "Queens Road",
+            "Kings Road",
+        ];
+        let mut rows: Vec<(String, String)> = Vec::with_capacity(1_000_000);
+        for i in 0..1_000_000usize {
+            // Vary the name so the index resembles ~1M distinct (street, area) rows.
+            rows.push((
+                format!("{} {}", roads[i % roads.len()], i % 4000),
+                "street".into(),
+            ));
+        }
+        rows.push(("London".into(), "city".into()));
+        let pd = PlaceData::from_names(&rows);
+
+        let start = std::time::Instant::now();
+        let mut hits = 0usize;
+        for _ in 0..50 {
+            let candidates = pd.place_candidate_rows(&["high", "street"]);
+            for row in candidates {
+                let idx = row as usize;
+                if place_search_test_score(&pd, idx, "high street", &["high", "street"]).is_some() {
+                    hits += 1;
+                }
+            }
+        }
+        let per_query = start.elapsed() / 50;
+        println!(
+            "indexed place search over {} rows: {:?}/query ({} hits)",
+            pd.name.len(),
+            per_query,
+            hits / 50
+        );
+        // The old full O(N) scan measured ~36ms here; candidate-based must be far under that.
+        assert!(per_query.as_millis() < 10, "per_query was {per_query:?}");
+    }
+
+    /// Mirrors the route's per-candidate match check for the bench.
+    fn place_search_test_score(
+        pd: &PlaceData,
+        idx: usize,
+        query_search: &str,
+        query_tokens: &[&str],
+    ) -> Option<f32> {
+        let search_text = &pd.name_search[idx];
+        if query_tokens.iter().all(|qt| {
+            place_alias_tokens(search_text)
+                .any(|t| t == *qt || (qt.len() >= 2 && t.starts_with(qt)))
+        }) {
+            Some(640.0)
+        } else if pd.name_lower[idx] == query_search {
+            Some(1000.0)
+        } else {
+            None
+        }
+    }
+
+    #[test]
+    fn fuzzy_candidate_rows_finds_typos_only_for_eligible_rows() {
+        let pd = PlaceData::from_names(&[
+            ("London", "city"),
+            ("Baker Street", "street"), // not fuzzy-eligible
+        ]);
+        let typo = compute_trigrams("Londn");
+        let candidates = pd.fuzzy_candidate_rows(&typo);
+        assert!(candidates.contains(&0)); // London (city) is reachable by fuzzy
+        assert!(!candidates.contains(&1)); // streets are excluded from the fuzzy index
+    }
+
     fn test_city_rows() -> [(&'static str, f32, f32, u32); 5] {
         [
             ("London", 51.507_446, -0.1277653, 8_908_083),
@@ -470,6 +910,29 @@ mod tests {
         assert!(build_search_text("Shadwell DLR station", "station").contains("shadwell station"));
     }
 
+    #[test]
+    fn search_text_expands_directional_and_size_abbreviations() {
+        assert!(build_search_text("Great Missenden", "village").contains("gt missenden"));
+        assert!(build_search_text("Mount Pleasant", "suburb").contains("mt pleasant"));
+        assert!(build_search_text("Little Venice", "suburb").contains("lt venice"));
+    }
+
+    #[test]
+    fn trigram_similarity_is_high_for_typos_and_low_for_unrelated() {
+        let london = compute_trigrams("London");
+        let typo = compute_trigrams("Londn");
+        let other = compute_trigrams("Manchester");
+        assert!(trigram_similarity(&london, &typo) >= 0.4);
+        assert!(trigram_similarity(&london, &other) < 0.2);
+        assert!((trigram_similarity(&london, &london) - 1.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn place_alias_tokens_split_across_aliases() {
+        let tokens: Vec<&str> = place_alias_tokens("kings cross | kings x").collect();
+        assert_eq!(tokens, vec!["kings", "cross", "kings", "x"]);
+    }
+
     #[test]
     fn travel_destination_types_match_legacy_places() {
         assert!(is_travel_destination_type("city"));