1066 lines
36 KiB
Rust
1066 lines
36 KiB
Rust
//! Address search: tokenization, query parsing, inverted/prefix indexes and the
|
||
//! ranked per-row search over property addresses.
|
||
|
||
use rustc_hash::{FxHashMap, FxHashSet};
|
||
|
||
use super::PropertyData;
|
||
|
||
/// Upper bound on rows scored per query. Intersection keeps most candidate sets far below
|
||
/// this; only a single very common road word (e.g. "high") approaches it, and the in-area
|
||
/// priority sort keeps a refined query's matches ahead of the cut.
|
||
const ADDRESS_SEARCH_CANDIDATE_LIMIT: usize = 150_000;
|
||
const ADDRESS_SEARCH_PREFIX_MIN_LEN: usize = 4;
|
||
const ADDRESS_SEARCH_PREFIX_MAX_LEN: usize = 8;
|
||
|
||
#[derive(Clone, Debug)]
|
||
pub(super) struct AddressTermGroup {
|
||
alternatives: Vec<String>,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
pub(super) struct AddressQuery {
|
||
full_postcode: Option<String>,
|
||
/// Compact uppercase outward code (optionally with a sector digit) recovered when the
|
||
/// user appended a partial postcode like "NW1" or "NW1 6". Used as an additive ranking
|
||
/// bias, never as a hard filter — so the disambiguating hint is honoured without
|
||
/// excluding the same road in other areas.
|
||
postcode_area: Option<String>,
|
||
text_groups: Vec<AddressTermGroup>,
|
||
numeric_terms: Vec<String>,
|
||
candidate_terms: Vec<String>,
|
||
}
|
||
|
||
fn tokenize_address_text(text: &str) -> Vec<String> {
|
||
let mut tokens = Vec::new();
|
||
let mut current = String::new();
|
||
|
||
for ch in text.chars() {
|
||
if ch.is_ascii_alphanumeric() {
|
||
current.push(ch.to_ascii_lowercase());
|
||
} else if crate::data::is_apostrophe(ch) {
|
||
continue;
|
||
} else if !current.is_empty() {
|
||
tokens.push(std::mem::take(&mut current));
|
||
}
|
||
}
|
||
|
||
if !current.is_empty() {
|
||
tokens.push(current);
|
||
}
|
||
|
||
tokens
|
||
}
|
||
|
||
fn is_full_postcode_compact(compact: &str) -> bool {
|
||
let bytes = compact.as_bytes();
|
||
let len = bytes.len();
|
||
if !(5..=7).contains(&len) {
|
||
return false;
|
||
}
|
||
|
||
let inward = &bytes[len - 3..];
|
||
if !inward[0].is_ascii_digit()
|
||
|| !inward[1].is_ascii_alphabetic()
|
||
|| !inward[2].is_ascii_alphabetic()
|
||
{
|
||
return false;
|
||
}
|
||
|
||
let outward = &bytes[..len - 3];
|
||
if !(2..=4).contains(&outward.len()) {
|
||
return false;
|
||
}
|
||
|
||
outward[0].is_ascii_alphabetic()
|
||
&& outward.iter().all(u8::is_ascii_alphanumeric)
|
||
&& outward.iter().any(u8::is_ascii_digit)
|
||
}
|
||
|
||
fn canonical_postcode_from_compact(compact: &str) -> String {
|
||
let upper = compact.to_ascii_uppercase();
|
||
let split = upper.len() - 3;
|
||
format!("{} {}", &upper[..split], &upper[split..])
|
||
}
|
||
|
||
fn extract_full_postcode(tokens: &[String]) -> Option<(String, Vec<usize>)> {
|
||
for (idx, token) in tokens.iter().enumerate() {
|
||
let compact = token.to_ascii_uppercase();
|
||
if is_full_postcode_compact(&compact) {
|
||
return Some((canonical_postcode_from_compact(&compact), vec![idx]));
|
||
}
|
||
}
|
||
|
||
for idx in 0..tokens.len().saturating_sub(1) {
|
||
let compact = format!(
|
||
"{}{}",
|
||
tokens[idx].to_ascii_uppercase(),
|
||
tokens[idx + 1].to_ascii_uppercase()
|
||
);
|
||
if is_full_postcode_compact(&compact) {
|
||
return Some((
|
||
canonical_postcode_from_compact(&compact),
|
||
vec![idx, idx + 1],
|
||
));
|
||
}
|
||
}
|
||
|
||
None
|
||
}
|
||
|
||
fn looks_like_postcode_fragment(token: &str) -> bool {
|
||
(2..=4).contains(&token.len())
|
||
&& token
|
||
.chars()
|
||
.next()
|
||
.is_some_and(|ch| ch.is_ascii_alphabetic())
|
||
&& token.chars().any(|ch| ch.is_ascii_digit())
|
||
&& token.chars().all(|ch| ch.is_ascii_alphanumeric())
|
||
}
|
||
|
||
fn is_numeric_address_token(token: &str) -> bool {
|
||
token.chars().all(|ch| ch.is_ascii_digit())
|
||
}
|
||
|
||
fn address_token_aliases(token: &str) -> Vec<&'static str> {
|
||
match token {
|
||
"apt" => vec!["apt", "apartment"],
|
||
"apartment" => vec!["apartment", "apt"],
|
||
"ave" => vec!["ave", "avenue"],
|
||
"avenue" => vec!["avenue", "ave"],
|
||
"blvd" => vec!["blvd", "boulevard"],
|
||
"boulevard" => vec!["boulevard", "blvd"],
|
||
"cl" => vec!["cl", "close"],
|
||
"close" => vec!["close", "cl"],
|
||
"ct" => vec!["ct", "court"],
|
||
"court" => vec!["court", "ct"],
|
||
"cres" => vec!["cres", "crescent"],
|
||
"crescent" => vec!["crescent", "cres"],
|
||
"dr" => vec!["dr", "drive"],
|
||
"drive" => vec!["drive", "dr"],
|
||
"fl" => vec!["fl", "flat"],
|
||
"flat" => vec!["flat", "fl"],
|
||
"gdns" => vec!["gdns", "gardens", "garden"],
|
||
"garden" => vec!["garden", "gardens", "gdns"],
|
||
"gardens" => vec!["gardens", "garden", "gdns"],
|
||
"hse" => vec!["hse", "house"],
|
||
"house" => vec!["house", "hse"],
|
||
"ln" => vec!["ln", "lane"],
|
||
"lane" => vec!["lane", "ln"],
|
||
"rd" => vec!["rd", "road"],
|
||
"road" => vec!["road", "rd"],
|
||
"sq" => vec!["sq", "square"],
|
||
"square" => vec!["square", "sq"],
|
||
"st" => vec!["st", "street", "saint"],
|
||
"street" => vec!["street", "st"],
|
||
"saint" => vec!["saint", "st"],
|
||
"terr" => vec!["terr", "terrace"],
|
||
"terrace" => vec!["terrace", "terr"],
|
||
_ => Vec::new(),
|
||
}
|
||
}
|
||
|
||
fn is_address_stop_token(token: &str) -> bool {
|
||
matches!(
|
||
token,
|
||
"a" | "an"
|
||
| "and"
|
||
| "apartment"
|
||
| "apt"
|
||
| "avenue"
|
||
| "ave"
|
||
| "block"
|
||
| "building"
|
||
| "bungalow"
|
||
| "close"
|
||
| "cl"
|
||
| "court"
|
||
| "ct"
|
||
| "cres"
|
||
| "crescent"
|
||
| "drive"
|
||
| "dr"
|
||
| "estate"
|
||
| "flat"
|
||
| "fl"
|
||
| "floor"
|
||
| "garden"
|
||
| "gardens"
|
||
| "gdns"
|
||
| "grove"
|
||
| "house"
|
||
| "hse"
|
||
| "lane"
|
||
| "ln"
|
||
| "lodge"
|
||
| "mansions"
|
||
| "mews"
|
||
| "of"
|
||
| "park"
|
||
| "place"
|
||
| "road"
|
||
| "rd"
|
||
| "room"
|
||
| "row"
|
||
| "saint"
|
||
| "sq"
|
||
| "square"
|
||
| "st"
|
||
| "street"
|
||
| "terr"
|
||
| "terrace"
|
||
| "the"
|
||
| "unit"
|
||
| "view"
|
||
| "villas"
|
||
| "walk"
|
||
| "way"
|
||
| "yard"
|
||
)
|
||
}
|
||
|
||
fn address_term_group(token: &str) -> Option<AddressTermGroup> {
|
||
if token.len() < 3 || is_numeric_address_token(token) || looks_like_postcode_fragment(token) {
|
||
return None;
|
||
}
|
||
|
||
let mut alternatives = Vec::new();
|
||
alternatives.push(token.to_string());
|
||
for alias in address_token_aliases(token) {
|
||
if !alternatives.iter().any(|existing| existing == alias) {
|
||
alternatives.push(alias.to_string());
|
||
}
|
||
}
|
||
|
||
if alternatives
|
||
.iter()
|
||
.all(|alternative| is_address_stop_token(alternative))
|
||
{
|
||
return None;
|
||
}
|
||
|
||
Some(AddressTermGroup { alternatives })
|
||
}
|
||
|
||
pub(super) fn address_search_tokens(text: &str) -> Vec<String> {
|
||
let mut tokens: Vec<String> = tokenize_address_text(text)
|
||
.into_iter()
|
||
.filter(|token| is_address_search_token(token))
|
||
.collect();
|
||
tokens.sort_unstable();
|
||
tokens.dedup();
|
||
tokens
|
||
}
|
||
|
||
/// Search tokens for a property row, covering its display address plus an
|
||
/// alternative spelling (`alt`, the EPC form) when distinct — so the property is
|
||
/// findable by either form. The result is deduped: each token appears at most
|
||
/// once, which the inverted-index posting lists rely on (a token must post a
|
||
/// given row exactly once to keep those lists strictly ascending and unique).
|
||
pub(super) fn merged_address_search_tokens(display: &str, alt: Option<&str>) -> Vec<String> {
|
||
let mut tokens = address_search_tokens(display);
|
||
if let Some(alt) = alt {
|
||
let alt = alt.trim();
|
||
if !alt.is_empty() && alt != display.trim() {
|
||
for token in address_search_tokens(alt) {
|
||
if !tokens.contains(&token) {
|
||
tokens.push(token);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
tokens
|
||
}
|
||
|
||
fn is_address_search_token(token: &str) -> bool {
|
||
if looks_like_postcode_fragment(token) {
|
||
return false;
|
||
}
|
||
|
||
if is_numeric_address_token(token) {
|
||
return true;
|
||
}
|
||
|
||
if token.chars().any(|ch| ch.is_ascii_digit()) {
|
||
return token.len() >= 2;
|
||
}
|
||
|
||
token.len() >= 3
|
||
}
|
||
|
||
pub(super) fn is_address_candidate_token(token: &str) -> bool {
|
||
!is_numeric_address_token(token)
|
||
&& !looks_like_postcode_fragment(token)
|
||
&& (token.chars().any(|ch| ch.is_ascii_digit())
|
||
|| (token.len() >= 3 && !is_address_stop_token(token)))
|
||
}
|
||
|
||
fn address_prefix_key(term: &str) -> &str {
|
||
if term.len() > ADDRESS_SEARCH_PREFIX_MAX_LEN {
|
||
&term[..ADDRESS_SEARCH_PREFIX_MAX_LEN]
|
||
} else {
|
||
term
|
||
}
|
||
}
|
||
|
||
pub(super) fn build_address_prefix_index(
|
||
address_token_index: &FxHashMap<String, Vec<u32>>,
|
||
) -> FxHashMap<String, Vec<String>> {
|
||
let mut prefix_index: FxHashMap<String, Vec<String>> = FxHashMap::default();
|
||
|
||
for token in address_token_index.keys() {
|
||
let max_prefix_len = token.len().min(ADDRESS_SEARCH_PREFIX_MAX_LEN);
|
||
for prefix_len in ADDRESS_SEARCH_PREFIX_MIN_LEN..=max_prefix_len {
|
||
prefix_index
|
||
.entry(token[..prefix_len].to_string())
|
||
.or_default()
|
||
.push(token.clone());
|
||
}
|
||
}
|
||
|
||
for tokens in prefix_index.values_mut() {
|
||
tokens.sort_unstable();
|
||
tokens.dedup();
|
||
tokens.shrink_to_fit();
|
||
}
|
||
|
||
prefix_index
|
||
}
|
||
|
||
/// Intersect two ascending-sorted row-id slices.
|
||
fn intersect_sorted(left: &[u32], right: &[u32]) -> Vec<u32> {
|
||
let mut out = Vec::new();
|
||
let (mut i, mut j) = (0, 0);
|
||
while i < left.len() && j < right.len() {
|
||
match left[i].cmp(&right[j]) {
|
||
std::cmp::Ordering::Less => i += 1,
|
||
std::cmp::Ordering::Greater => j += 1,
|
||
std::cmp::Ordering::Equal => {
|
||
out.push(left[i]);
|
||
i += 1;
|
||
j += 1;
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Union two ascending-sorted row-id slices (deduplicated, stays sorted).
|
||
fn union_sorted(left: &[u32], right: &[u32]) -> Vec<u32> {
|
||
let mut out = Vec::with_capacity(left.len() + right.len());
|
||
let (mut i, mut j) = (0, 0);
|
||
while i < left.len() && j < right.len() {
|
||
match left[i].cmp(&right[j]) {
|
||
std::cmp::Ordering::Less => {
|
||
out.push(left[i]);
|
||
i += 1;
|
||
}
|
||
std::cmp::Ordering::Greater => {
|
||
out.push(right[j]);
|
||
j += 1;
|
||
}
|
||
std::cmp::Ordering::Equal => {
|
||
out.push(left[i]);
|
||
i += 1;
|
||
j += 1;
|
||
}
|
||
}
|
||
}
|
||
out.extend_from_slice(&left[i..]);
|
||
out.extend_from_slice(&right[j..]);
|
||
out
|
||
}
|
||
|
||
/// An ordinal like "1st", "2nd", "3rd", "21st" — part of the street name ("2nd Avenue"), not a
|
||
/// house-number prefix.
|
||
fn is_ordinal_token(token: &str) -> bool {
|
||
let split = token.len().saturating_sub(2);
|
||
let (digits, suffix) = token.split_at(split);
|
||
!digits.is_empty()
|
||
&& digits.chars().all(|ch| ch.is_ascii_digit())
|
||
&& matches!(suffix, "st" | "nd" | "rd" | "th")
|
||
}
|
||
|
||
/// Leading address tokens that denote a unit/house number rather than the street itself.
|
||
fn is_house_prefix_token(token: &str) -> bool {
|
||
if is_ordinal_token(token) {
|
||
return false;
|
||
}
|
||
matches!(
|
||
token,
|
||
"flat" | "fl" | "apartment" | "apt" | "unit" | "no" | "block" | "floor" | "room"
|
||
) || token.len() == 1
|
||
|| token.chars().all(|ch| ch.is_ascii_digit())
|
||
|| (token.chars().next().is_some_and(|ch| ch.is_ascii_digit())
|
||
&& token.chars().any(|ch| ch.is_ascii_alphabetic()))
|
||
}
|
||
|
||
/// Street-level key for an address: drops the leading house-number / flat prefix so that
|
||
/// "12 Baker Street" and "5 Baker Street" collapse to a single street entry.
|
||
fn street_key(address: &str) -> String {
|
||
let tokens = tokenize_address_text(address);
|
||
let mut start = 0;
|
||
while start < tokens.len() && is_house_prefix_token(&tokens[start]) {
|
||
start += 1;
|
||
}
|
||
if start >= tokens.len() {
|
||
return tokens.join(" ");
|
||
}
|
||
tokens[start..].join(" ")
|
||
}
|
||
|
||
/// Road-type words. Their presence (with no house number) marks a road browse, which we
|
||
/// collapse to one result per street.
|
||
const ROAD_TYPE_TOKENS: &[&str] = &[
|
||
"street",
|
||
"st",
|
||
"road",
|
||
"rd",
|
||
"lane",
|
||
"ln",
|
||
"avenue",
|
||
"ave",
|
||
"close",
|
||
"cl",
|
||
"drive",
|
||
"dr",
|
||
"way",
|
||
"court",
|
||
"ct",
|
||
"crescent",
|
||
"cres",
|
||
"place",
|
||
"terrace",
|
||
"terr",
|
||
"grove",
|
||
"gardens",
|
||
"gdns",
|
||
"walk",
|
||
"row",
|
||
"square",
|
||
"sq",
|
||
"hill",
|
||
"parade",
|
||
"mews",
|
||
"embankment",
|
||
"broadway",
|
||
"boulevard",
|
||
"blvd",
|
||
];
|
||
|
||
fn query_has_road_type(query: &str) -> bool {
|
||
tokenize_address_text(query)
|
||
.iter()
|
||
.any(|token| ROAD_TYPE_TOKENS.contains(&token.as_str()))
|
||
}
|
||
|
||
/// The outward code (everything before the space) of a canonical postcode.
|
||
fn outcode_of(postcode: &str) -> &str {
|
||
postcode.split(' ').next().unwrap_or(postcode)
|
||
}
|
||
|
||
fn parse_address_query(query: &str) -> AddressQuery {
|
||
let tokens = tokenize_address_text(query);
|
||
let (full_postcode, postcode_token_indices) = extract_full_postcode(&tokens)
|
||
.map(|(postcode, indices)| (Some(postcode), indices))
|
||
.unwrap_or((None, Vec::new()));
|
||
|
||
let skip_postcode_tokens: FxHashSet<usize> = postcode_token_indices.into_iter().collect();
|
||
|
||
// Recover an appended partial postcode (outcode, or outcode + sector digit) as a ranking
|
||
// bias rather than discarding it — but only from the TRAILING position, so a leading road
|
||
// designation like "A4 Great West Road" is not mistaken for an area refinement.
|
||
let mut postcode_area: Option<String> = None;
|
||
let mut consumed_partial_tokens: FxHashSet<usize> = FxHashSet::default();
|
||
if full_postcode.is_none() && !tokens.is_empty() {
|
||
let last = tokens.len() - 1;
|
||
if !skip_postcode_tokens.contains(&last) {
|
||
let sector_digit =
|
||
tokens[last].len() == 1 && tokens[last].chars().all(|ch| ch.is_ascii_digit());
|
||
if last >= 1
|
||
&& sector_digit
|
||
&& !skip_postcode_tokens.contains(&(last - 1))
|
||
&& looks_like_postcode_fragment(&tokens[last - 1])
|
||
{
|
||
postcode_area = Some(format!(
|
||
"{}{}",
|
||
tokens[last - 1].to_ascii_uppercase(),
|
||
tokens[last]
|
||
));
|
||
consumed_partial_tokens.insert(last);
|
||
consumed_partial_tokens.insert(last - 1);
|
||
} else if looks_like_postcode_fragment(&tokens[last]) {
|
||
postcode_area = Some(tokens[last].to_ascii_uppercase());
|
||
consumed_partial_tokens.insert(last);
|
||
}
|
||
}
|
||
}
|
||
|
||
let mut text_groups = Vec::new();
|
||
let mut numeric_terms = Vec::new();
|
||
let mut candidate_terms = Vec::new();
|
||
|
||
for (idx, token) in tokens.iter().enumerate() {
|
||
if skip_postcode_tokens.contains(&idx)
|
||
|| consumed_partial_tokens.contains(&idx)
|
||
|| looks_like_postcode_fragment(token)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
if is_numeric_address_token(token) {
|
||
numeric_terms.push(token.clone());
|
||
continue;
|
||
}
|
||
|
||
if let Some(group) = address_term_group(token) {
|
||
for alternative in &group.alternatives {
|
||
if !is_address_stop_token(alternative)
|
||
&& !candidate_terms.iter().any(|term| term == alternative)
|
||
{
|
||
candidate_terms.push(alternative.clone());
|
||
}
|
||
}
|
||
text_groups.push(group);
|
||
} else if token.chars().any(|ch| ch.is_ascii_digit()) && token.len() >= 2 {
|
||
numeric_terms.push(token.clone());
|
||
if !candidate_terms.iter().any(|term| term == token) {
|
||
candidate_terms.push(token.clone());
|
||
}
|
||
}
|
||
}
|
||
|
||
text_groups.dedup_by(|left, right| left.alternatives == right.alternatives);
|
||
numeric_terms.sort_unstable();
|
||
numeric_terms.dedup();
|
||
|
||
AddressQuery {
|
||
full_postcode,
|
||
postcode_area,
|
||
text_groups,
|
||
numeric_terms,
|
||
candidate_terms,
|
||
}
|
||
}
|
||
|
||
fn token_matches_query_term(token: &str, query_term: &str) -> bool {
|
||
token == query_term || (query_term.len() >= 3 && token.starts_with(query_term))
|
||
}
|
||
|
||
fn token_matches_numeric_term(token: &str, query_term: &str) -> bool {
|
||
token == query_term || token.starts_with(query_term)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
fn address_tokens_match_group(tokens: &[String], group: &AddressTermGroup) -> bool {
|
||
group.alternatives.iter().any(|alternative| {
|
||
tokens
|
||
.iter()
|
||
.any(|token| token_matches_query_term(token, alternative))
|
||
})
|
||
}
|
||
|
||
impl PropertyData {
|
||
fn row_address_search_tokens(&self, row: usize) -> &[lasso::Spur] {
|
||
let offset = self.address_search_token_offsets[row] as usize;
|
||
let length = self.address_search_token_lengths[row] as usize;
|
||
&self.address_search_token_keys[offset..offset + length]
|
||
}
|
||
|
||
/// Search individual property addresses, returning `(row, score)` ranked best-first.
|
||
///
|
||
/// Candidate rows come from intersecting the posting lists of the distinctive words the
|
||
/// user typed in full (so "Cherry Hinton Road" narrows to rows containing both), unioned
|
||
/// with the exact-postcode rows when a complete postcode is present (so a postcode is a
|
||
/// boost, not an all-or-nothing gate). An appended partial postcode keeps in-area rows
|
||
/// ahead of the candidate cut and adds a scoring bias. With a road-type word and no house
|
||
/// number, results collapse to one row per street.
|
||
pub fn search_addresses(&self, query: &str, limit: usize) -> Vec<(usize, i32)> {
|
||
if limit == 0 {
|
||
return Vec::new();
|
||
}
|
||
|
||
let parsed = parse_address_query(query);
|
||
if parsed.full_postcode.is_none()
|
||
&& parsed.text_groups.is_empty()
|
||
&& parsed.numeric_terms.is_empty()
|
||
{
|
||
return Vec::new();
|
||
}
|
||
|
||
let mut candidate_rows = self.address_candidate_rows(&parsed.candidate_terms);
|
||
|
||
// A complete postcode contributes its rows too, instead of replacing the road match.
|
||
if let Some(postcode) = parsed.full_postcode.as_deref() {
|
||
if let Some(rows) = self
|
||
.postcode_interner
|
||
.get(postcode)
|
||
.and_then(|key| self.postcode_row_index.get(&key))
|
||
{
|
||
candidate_rows = if candidate_rows.is_empty() {
|
||
rows.clone()
|
||
} else {
|
||
union_sorted(&candidate_rows, rows)
|
||
};
|
||
}
|
||
}
|
||
|
||
if candidate_rows.is_empty() {
|
||
return Vec::new();
|
||
}
|
||
|
||
// When the user appended a partial postcode, keep in-area rows ahead of the cut so the
|
||
// refinement still surfaces even for very common roads. Single pass (stable partition) so
|
||
// the postcode check — which allocates — runs exactly once per candidate.
|
||
if let Some(area) = parsed.postcode_area.as_deref() {
|
||
let mut in_area = Vec::new();
|
||
let mut others = Vec::new();
|
||
for &row in &candidate_rows {
|
||
if self.row_postcode_in_area(row as usize, area) {
|
||
in_area.push(row);
|
||
} else {
|
||
others.push(row);
|
||
}
|
||
}
|
||
in_area.extend(others);
|
||
candidate_rows = in_area;
|
||
}
|
||
candidate_rows.truncate(ADDRESS_SEARCH_CANDIDATE_LIMIT);
|
||
|
||
let mut scored: Vec<(i32, usize, usize)> = candidate_rows
|
||
.into_iter()
|
||
.filter_map(|row| {
|
||
let row = row as usize;
|
||
self.address_match_score(row, &parsed)
|
||
.map(|score| (score, self.address(row).len(), row))
|
||
})
|
||
.collect();
|
||
|
||
scored.sort_unstable_by(|left, right| {
|
||
right
|
||
.0
|
||
.cmp(&left.0)
|
||
.then(left.1.cmp(&right.1))
|
||
.then(left.2.cmp(&right.2))
|
||
});
|
||
|
||
// Collapse a road browse (road-type word, no house number) to one row per street.
|
||
let collapse_streets = parsed.numeric_terms.is_empty() && query_has_road_type(query);
|
||
|
||
let mut seen = FxHashSet::default();
|
||
let mut results = Vec::with_capacity(limit);
|
||
for (score, _, row) in scored {
|
||
let address = self.address(row).trim();
|
||
if address.is_empty() {
|
||
continue;
|
||
}
|
||
let key = if collapse_streets {
|
||
format!(
|
||
"{}\n{}",
|
||
street_key(address),
|
||
outcode_of(self.postcode(row))
|
||
)
|
||
} else {
|
||
format!("{}\n{}", address.to_ascii_lowercase(), self.postcode(row))
|
||
};
|
||
if !seen.insert(key) {
|
||
continue;
|
||
}
|
||
results.push((row, score));
|
||
if results.len() == limit {
|
||
break;
|
||
}
|
||
}
|
||
|
||
results
|
||
}
|
||
|
||
/// True when the row's postcode begins with the compact partial-postcode `area`
|
||
/// (e.g. "NW1" or "NW16" matches "NW1 6XE").
|
||
fn row_postcode_in_area(&self, row: usize, area: &str) -> bool {
|
||
let mut compact = String::new();
|
||
for ch in self.postcode(row).chars() {
|
||
if !ch.is_whitespace() {
|
||
compact.push(ch.to_ascii_uppercase());
|
||
}
|
||
}
|
||
compact.starts_with(area)
|
||
}
|
||
|
||
/// Candidate rows for the distinctive query words. Words typed in full intersect by their
|
||
/// exact posting lists (precise); a still-being-typed final word with no exact match seeds
|
||
/// from the smallest prefix-expanded posting list (so partial typing keeps working).
|
||
fn address_candidate_rows(&self, terms: &[String]) -> Vec<u32> {
|
||
let mut exact: Vec<&[u32]> = terms
|
||
.iter()
|
||
.filter_map(|term| self.address_token_index.get(term).map(Vec::as_slice))
|
||
.collect();
|
||
|
||
if !exact.is_empty() {
|
||
exact.sort_by_key(|rows| rows.len());
|
||
let mut acc = exact[0].to_vec();
|
||
for rows in &exact[1..] {
|
||
if acc.is_empty() {
|
||
break;
|
||
}
|
||
acc = intersect_sorted(&acc, rows);
|
||
}
|
||
return acc;
|
||
}
|
||
|
||
self.prefix_seed_rows(terms)
|
||
}
|
||
|
||
/// Seed rows from the smallest prefix-expanded term — used only when no word matched an
|
||
/// indexed token exactly (i.e. the user is still typing the final word).
|
||
fn prefix_seed_rows(&self, terms: &[String]) -> Vec<u32> {
|
||
let mut best: Option<Vec<u32>> = None;
|
||
for term in terms {
|
||
if term.len() < ADDRESS_SEARCH_PREFIX_MIN_LEN {
|
||
continue;
|
||
}
|
||
let Some(tokens) = self.address_prefix_index.get(address_prefix_key(term)) else {
|
||
continue;
|
||
};
|
||
let mut union: Vec<u32> = Vec::new();
|
||
for token in tokens {
|
||
if !token.starts_with(term) {
|
||
continue;
|
||
}
|
||
if let Some(rows) = self.address_token_index.get(token) {
|
||
union = if union.is_empty() {
|
||
rows.clone()
|
||
} else {
|
||
union_sorted(&union, rows)
|
||
};
|
||
}
|
||
}
|
||
if !union.is_empty()
|
||
&& best
|
||
.as_ref()
|
||
.is_none_or(|current| union.len() < current.len())
|
||
{
|
||
best = Some(union);
|
||
}
|
||
}
|
||
best.unwrap_or_default()
|
||
}
|
||
|
||
fn address_match_score(&self, row: usize, parsed: &AddressQuery) -> Option<i32> {
|
||
if self.address(row).trim().is_empty() {
|
||
return None;
|
||
}
|
||
|
||
let tokens = self.row_address_search_tokens(row);
|
||
if parsed
|
||
.text_groups
|
||
.iter()
|
||
.any(|group| !self.address_tokens_match_group(tokens, group))
|
||
{
|
||
return None;
|
||
}
|
||
|
||
let numeric_matches = parsed
|
||
.numeric_terms
|
||
.iter()
|
||
.filter(|term| {
|
||
tokens.iter().any(|token| {
|
||
token_matches_numeric_term(self.address_search_interner.resolve(token), term)
|
||
})
|
||
})
|
||
.count();
|
||
|
||
if !parsed.numeric_terms.is_empty() && numeric_matches == 0 {
|
||
return None;
|
||
}
|
||
|
||
let mut score = 0;
|
||
if parsed.full_postcode.is_some() {
|
||
score += 1_000;
|
||
}
|
||
score += (parsed.text_groups.len() as i32) * 200;
|
||
score += (numeric_matches as i32) * 90;
|
||
if numeric_matches == parsed.numeric_terms.len() && numeric_matches > 0 {
|
||
score += 50;
|
||
}
|
||
// Additive bias (never a filter) when the row sits in the appended partial postcode.
|
||
if let Some(area) = parsed.postcode_area.as_deref() {
|
||
if self.row_postcode_in_area(row, area) {
|
||
score += 400;
|
||
}
|
||
}
|
||
|
||
Some(score)
|
||
}
|
||
|
||
fn address_tokens_match_group(&self, tokens: &[lasso::Spur], group: &AddressTermGroup) -> bool {
|
||
group.alternatives.iter().any(|alternative| {
|
||
tokens.iter().any(|token| {
|
||
token_matches_query_term(self.address_search_interner.resolve(token), alternative)
|
||
})
|
||
})
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn tokenize_address_text_elides_every_apostrophe_variant() {
|
||
// The query and the index both run through this tokenizer, so any apostrophe glyph must
|
||
// join the word ("O'Brien" -> "obrien") rather than split it; otherwise the same address
|
||
// tokenizes differently depending on which quote glyph was typed.
|
||
let expected = vec!["obrien".to_string(), "road".to_string()];
|
||
for addr in [
|
||
"O'Brien Road", // U+0027 straight
|
||
"O’Brien Road", // U+2019 right single quote
|
||
"O‘Brien Road", // U+2018 left single quote
|
||
"O´Brien Road", // U+00B4 acute accent
|
||
"OʼBrien Road", // U+02BC modifier letter apostrophe
|
||
"O′Brien Road", // U+2032 prime
|
||
] {
|
||
assert_eq!(tokenize_address_text(addr), expected, "failed for {addr:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn full_postcode_detection_accepts_common_formats() {
|
||
assert!(is_full_postcode_compact("SW1A1AA"));
|
||
assert!(is_full_postcode_compact("E142DG"));
|
||
assert!(is_full_postcode_compact("M11AE"));
|
||
assert!(!is_full_postcode_compact("E14"));
|
||
assert!(!is_full_postcode_compact("DOWNING"));
|
||
assert!(!is_full_postcode_compact("10A"));
|
||
}
|
||
|
||
#[test]
|
||
fn address_query_parsing_skips_postcodes_and_street_suffixes() {
|
||
let parsed = parse_address_query("Flat 2, 10 Downing St, SW1A 2AA");
|
||
|
||
assert_eq!(parsed.full_postcode.as_deref(), Some("SW1A 2AA"));
|
||
assert_eq!(
|
||
parsed.numeric_terms,
|
||
vec!["10".to_string(), "2".to_string()]
|
||
);
|
||
assert_eq!(parsed.candidate_terms, vec!["downing".to_string()]);
|
||
assert_eq!(parsed.text_groups.len(), 1);
|
||
assert_eq!(
|
||
parsed.text_groups[0].alternatives,
|
||
vec!["downing".to_string()]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn address_query_parsing_handles_compact_postcodes() {
|
||
let parsed = parse_address_query("10 downing street sw1a1aa");
|
||
|
||
assert_eq!(parsed.full_postcode.as_deref(), Some("SW1A 1AA"));
|
||
assert_eq!(parsed.numeric_terms, vec!["10".to_string()]);
|
||
assert_eq!(parsed.candidate_terms, vec!["downing".to_string()]);
|
||
}
|
||
|
||
#[test]
|
||
fn address_query_recovers_appended_partial_postcode_as_bias() {
|
||
let parsed = parse_address_query("Baker Street NW1");
|
||
assert_eq!(parsed.full_postcode, None);
|
||
assert_eq!(parsed.postcode_area.as_deref(), Some("NW1"));
|
||
// The road words are still searchable; the postcode fragment did not consume them.
|
||
assert_eq!(parsed.candidate_terms, vec!["baker".to_string()]);
|
||
assert!(parsed.numeric_terms.is_empty());
|
||
}
|
||
|
||
#[test]
|
||
fn address_query_recovers_outcode_plus_sector_without_a_phantom_house_number() {
|
||
let parsed = parse_address_query("High Street CR0 2");
|
||
assert_eq!(parsed.postcode_area.as_deref(), Some("CR02"));
|
||
// The lone sector digit must not be treated as a house number.
|
||
assert!(parsed.numeric_terms.is_empty());
|
||
assert_eq!(parsed.candidate_terms, vec!["high".to_string()]);
|
||
}
|
||
|
||
#[test]
|
||
fn full_postcode_takes_precedence_over_partial_bias() {
|
||
let parsed = parse_address_query("Baker Street NW1 6XE");
|
||
assert_eq!(parsed.full_postcode.as_deref(), Some("NW1 6XE"));
|
||
assert_eq!(parsed.postcode_area, None);
|
||
}
|
||
|
||
#[test]
|
||
fn intersect_and_union_sorted_row_ids() {
|
||
assert_eq!(
|
||
intersect_sorted(&[1, 2, 3, 5], &[2, 3, 4, 5]),
|
||
vec![2, 3, 5]
|
||
);
|
||
assert_eq!(intersect_sorted(&[1, 2], &[3, 4]), Vec::<u32>::new());
|
||
assert_eq!(union_sorted(&[1, 3, 5], &[2, 3, 4]), vec![1, 2, 3, 4, 5]);
|
||
assert_eq!(union_sorted(&[], &[2, 4]), vec![2, 4]);
|
||
}
|
||
|
||
#[test]
|
||
fn street_key_collapses_house_numbers_and_flats() {
|
||
assert_eq!(street_key("12 Baker Street"), "baker street");
|
||
assert_eq!(street_key("5 Baker Street"), "baker street");
|
||
assert_eq!(street_key("Flat 2, 10 Downing Street"), "downing street");
|
||
assert_eq!(street_key("221B Baker Street"), "baker street");
|
||
}
|
||
|
||
#[test]
|
||
fn street_key_keeps_ordinal_street_names() {
|
||
// Ordinals are part of the street name, not a house-number prefix.
|
||
assert_eq!(street_key("2nd Avenue"), "2nd avenue");
|
||
assert_eq!(street_key("12 3rd Avenue"), "3rd avenue");
|
||
assert!(is_ordinal_token("21st"));
|
||
assert!(!is_ordinal_token("21"));
|
||
assert!(!is_ordinal_token("221b"));
|
||
}
|
||
|
||
#[test]
|
||
fn postcode_area_recovered_only_from_the_trailing_position() {
|
||
// A leading road designation must NOT be taken as an area refinement.
|
||
let parsed = parse_address_query("A4 Great West Road");
|
||
assert_eq!(parsed.postcode_area, None);
|
||
// A genuine trailing outcode still is.
|
||
let trailing = parse_address_query("Great West Road W4");
|
||
assert_eq!(trailing.postcode_area.as_deref(), Some("W4"));
|
||
}
|
||
|
||
#[test]
|
||
fn road_type_detection() {
|
||
assert!(query_has_road_type("high street"));
|
||
assert!(query_has_road_type("acacia avenue"));
|
||
assert!(!query_has_road_type("acacia"));
|
||
assert!(!query_has_road_type("london"));
|
||
}
|
||
|
||
#[test]
|
||
fn address_query_parsing_keeps_partial_terms_for_row_matching() {
|
||
let parsed = parse_address_query("settlers cour");
|
||
|
||
assert_eq!(parsed.full_postcode, None);
|
||
assert_eq!(parsed.numeric_terms, Vec::<String>::new());
|
||
assert_eq!(
|
||
parsed.candidate_terms,
|
||
vec!["settlers".to_string(), "cour".to_string()]
|
||
);
|
||
assert_eq!(parsed.text_groups.len(), 2);
|
||
assert_eq!(
|
||
parsed.text_groups[0].alternatives,
|
||
vec!["settlers".to_string()]
|
||
);
|
||
assert_eq!(parsed.text_groups[1].alternatives, vec!["cour".to_string()]);
|
||
}
|
||
|
||
#[test]
|
||
fn merged_tokens_index_both_address_forms() {
|
||
// A property whose price-paid form is "10 Park Road" but whose EPC form
|
||
// names the building ("Kingswood House 10 Park Road") must be findable by
|
||
// the building name too: the distinctive "kingswood" token comes only
|
||
// from the EPC form.
|
||
let display_only = address_search_tokens("10 Park Road");
|
||
assert!(!display_only.contains(&"kingswood".to_string()));
|
||
|
||
let merged =
|
||
merged_address_search_tokens("10 Park Road", Some("Kingswood House 10 Park Road"));
|
||
assert!(merged.contains(&"kingswood".to_string()));
|
||
// Tokens shared by both forms are still present and not duplicated.
|
||
assert!(merged.contains(&"park".to_string()));
|
||
assert!(merged.contains(&"road".to_string()));
|
||
assert!(merged.contains(&"10".to_string()));
|
||
|
||
// No token repeats — posting-list uniqueness depends on this.
|
||
let mut deduped = merged.clone();
|
||
deduped.sort_unstable();
|
||
deduped.dedup();
|
||
assert_eq!(deduped.len(), merged.len());
|
||
}
|
||
|
||
#[test]
|
||
fn merged_tokens_ignore_redundant_or_missing_alt_form() {
|
||
let baseline = address_search_tokens("12 Baker Street");
|
||
// An EPC form identical to the display adds nothing.
|
||
assert_eq!(
|
||
merged_address_search_tokens("12 Baker Street", Some("12 Baker Street")),
|
||
baseline
|
||
);
|
||
// A blank EPC form (whitespace) and an absent one are both no-ops.
|
||
assert_eq!(
|
||
merged_address_search_tokens("12 Baker Street", Some(" ")),
|
||
baseline
|
||
);
|
||
assert_eq!(
|
||
merged_address_search_tokens("12 Baker Street", None),
|
||
baseline
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn merged_tokens_handle_epc_only_display() {
|
||
// EPC-only dwellings have no price-paid form, so the EPC address IS the
|
||
// display; passing it as both arguments must not duplicate its tokens.
|
||
let epc = "Flat 4 Rosewood Court";
|
||
assert_eq!(
|
||
merged_address_search_tokens(epc, Some(epc)),
|
||
address_search_tokens(epc)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn address_search_tokens_keep_actual_address_terms_for_scoring() {
|
||
let tokens = address_search_tokens("Flat 2, 10 Downing Cour");
|
||
|
||
assert_eq!(
|
||
tokens,
|
||
vec![
|
||
"10".to_string(),
|
||
"2".to_string(),
|
||
"cour".to_string(),
|
||
"downing".to_string(),
|
||
"flat".to_string()
|
||
]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn address_prefix_index_finds_partial_address_terms() {
|
||
let mut token_index: FxHashMap<String, Vec<u32>> = FxHashMap::default();
|
||
token_index.insert("downing".to_string(), vec![1]);
|
||
token_index.insert("downton".to_string(), vec![2]);
|
||
token_index.insert("market".to_string(), vec![3]);
|
||
|
||
let prefix_index = build_address_prefix_index(&token_index);
|
||
|
||
assert_eq!(
|
||
prefix_index.get("down").cloned().unwrap_or_default(),
|
||
vec!["downing".to_string(), "downton".to_string()]
|
||
);
|
||
assert_eq!(
|
||
prefix_index.get("downi").cloned().unwrap_or_default(),
|
||
vec!["downing".to_string()]
|
||
);
|
||
assert_eq!(
|
||
prefix_index.get("downt").cloned().unwrap_or_default(),
|
||
vec!["downton".to_string()]
|
||
);
|
||
assert!(!prefix_index.contains_key("do"));
|
||
}
|
||
|
||
#[test]
|
||
fn address_term_matching_allows_prefixes_and_aliases() {
|
||
let tokens = tokenize_address_text("10 Downing Street");
|
||
let prefix_group = address_term_group("down").expect("prefix term should be searchable");
|
||
let alias_group = AddressTermGroup {
|
||
alternatives: vec!["st".to_string(), "street".to_string()],
|
||
};
|
||
|
||
assert!(address_tokens_match_group(&tokens, &prefix_group));
|
||
assert!(address_tokens_match_group(&tokens, &alias_group));
|
||
}
|
||
|
||
#[test]
|
||
fn address_term_matching_uses_actual_token_prefixes() {
|
||
let tokens = tokenize_address_text("12 Settlers Court");
|
||
let prefix_group = address_term_group("cou").expect("partial term should be searchable");
|
||
|
||
assert!(address_tokens_match_group(&tokens, &prefix_group));
|
||
}
|
||
}
|