This commit is contained in:
Andras Schmelczer 2026-07-03 18:47:28 +01:00
parent ab688243d7
commit 463bd4c647
54 changed files with 13239 additions and 625 deletions

View file

@ -22,7 +22,7 @@ pub(super) struct AddressQuery {
full_postcode: Option<String>,
/// Compact uppercase outward code (optionally with a sector digit) recovered when the
/// user appended a partial postcode like "NW1" or "NW1 6". Used as an additive ranking
/// bias, never as a hard filter so the disambiguating hint is honoured without
/// bias, never as a hard filter, so the disambiguating hint is honoured without
/// excluding the same road in other areas.
postcode_area: Option<String>,
text_groups: Vec<AddressTermGroup>,
@ -252,7 +252,7 @@ pub(super) fn address_search_tokens(text: &str) -> Vec<String> {
}
/// Search tokens for a property row, covering its display address plus an
/// alternative spelling (`alt`, the EPC form) when distinct so the property is
/// alternative spelling (`alt`, the EPC form) when distinct, so the property is
/// findable by either form. The result is deduped: each token appears at most
/// once, which the inverted-index posting lists rely on (a token must post a
/// given row exactly once to keep those lists strictly ascending and unique).
@ -370,7 +370,7 @@ fn union_sorted(left: &[u32], right: &[u32]) -> Vec<u32> {
out
}
/// An ordinal like "1st", "2nd", "3rd", "21st" part of the street name ("2nd Avenue"), not a
/// An ordinal like "1st", "2nd", "3rd", "21st", part of the street name ("2nd Avenue"), not a
/// house-number prefix.
fn is_ordinal_token(token: &str) -> bool {
let split = token.len().saturating_sub(2);
@ -467,7 +467,7 @@ fn parse_address_query(query: &str) -> AddressQuery {
let skip_postcode_tokens: FxHashSet<usize> = postcode_token_indices.into_iter().collect();
// Recover an appended partial postcode (outcode, or outcode + sector digit) as a ranking
// bias rather than discarding it but only from the TRAILING position, so a leading road
// bias rather than discarding it, but only from the TRAILING position, so a leading road
// designation like "A4 Great West Road" is not mistaken for an area refinement.
let mut postcode_area: Option<String> = None;
let mut consumed_partial_tokens: FxHashSet<usize> = FxHashSet::default();
@ -610,7 +610,7 @@ impl PropertyData {
// When the user appended a partial postcode, keep in-area rows ahead of the cut so the
// refinement still surfaces even for very common roads. Single pass (stable partition) so
// the postcode check — which allocates — runs exactly once per candidate.
// the postcode check (which allocates) runs exactly once per candidate.
if let Some(area) = parsed.postcode_area.as_deref() {
let mut in_area = Vec::new();
let mut others = Vec::new();
@ -710,7 +710,7 @@ impl PropertyData {
self.prefix_seed_rows(terms)
}
/// Seed rows from the smallest prefix-expanded term used only when no word matched an
/// Seed rows from the smallest prefix-expanded term, used only when no word matched an
/// indexed token exactly (i.e. the user is still typing the final word).
fn prefix_seed_rows(&self, terms: &[String]) -> Vec<u32> {
let mut best: Option<Vec<u32>> = None;
@ -967,7 +967,7 @@ mod tests {
assert!(merged.contains(&"road".to_string()));
assert!(merged.contains(&"10".to_string()));
// No token repeats posting-list uniqueness depends on this.
// No token repeats: posting-list uniqueness depends on this.
let mut deduped = merged.clone();
deduped.sort_unstable();
deduped.dedup();

View file

@ -1,13 +1,13 @@
//! Optional disk-backed storage for the large flat arrays in [`PropertyData`].
//!
//! In production every property array is held as an owned `Vec` in RAM — fastest,
//! In production every property array is held as an owned `Vec` in RAM (fastest),
//! but the feature matrix plus the flat address-search arrays are ~4GB, which a
//! memory-constrained dev box can't hold reliably. When a spill directory is
//! configured (the `--spill-dir` dev flag), each large array is instead written
//! to an anonymous file in that directory and memory-mapped read-only. The mapped
//! pages are file-backed and clean, so under memory pressure the kernel evicts
//! them (re-faulting from disk on next touch) rather than the process being
//! OOM-killed the same "let the kernel page it" trade-off the PMTiles reader
//! OOM-killed: the same "let the kernel page it" trade-off the PMTiles reader
//! already makes. The backing files are unlinked immediately after creation, so
//! they leave nothing on disk and are reclaimed when the map drops at shutdown.
//!
@ -28,8 +28,8 @@ use memmap2::{Mmap, MmapMut};
/// round-trip losslessly through their raw bytes.
///
/// # Safety
/// Implementors must be `Copy`, contain no padding, and accept for any value we
/// could have stored the exact bytes we wrote when reinterpreted as `Self`. We
/// Implementors must be `Copy`, contain no padding, and accept (for any value we
/// could have stored) the exact bytes we wrote when reinterpreted as `Self`. We
/// only ever read back bytes produced from genuine `Self` values, so niche types
/// such as `lasso::Spur` (a `NonZeroU32`) are sound here: the bytes always come
/// from real, non-zero interner keys and are never reinterpreted from arbitrary
@ -86,7 +86,7 @@ impl<T: SpillElem> SpillVec<T> {
}
impl SpillVec<u16> {
/// Build a `u16` array of `len` elements by filling it in place on the heap
/// Build a `u16` array of `len` elements by filling it in place: on the heap
/// when `dir` is `None`, or directly inside an mmap-backed spill file when
/// `Some`, so the (large) buffer never simultaneously exists on the heap. Every
/// element is initialized to `default` first; `fill` then overwrites the cells
@ -138,7 +138,7 @@ impl SpillVec<u16> {
///
/// When a spill `dir` is set (and the length is non-zero) the backing store is a
/// pre-sized, memory-mapped file and each pushed element is written straight into
/// it so the array never exists as a heap `Vec` and is never copied a second
/// it, so the array never exists as a heap `Vec` and is never copied a second
/// time on finalisation, unlike [`SpillVec::maybe_spill`], which takes an
/// already-built `Vec` and so needs the whole thing resident first. Without a
/// spill dir it accumulates into an owned `Vec` (production behaviour, identical
@ -200,7 +200,7 @@ impl<T: SpillElem> SpillVecBuilder<T> {
}
/// Append one element. Panics if more than the declared `len` elements are
/// pushed — enforced identically on both backings so a streaming bug fails
/// pushed. Enforced identically on both backings so a streaming bug fails
/// the same way in production (owned) as in dev (spilled).
#[inline]
pub fn push(&mut self, value: T) {
@ -320,7 +320,7 @@ fn spill_vec<T: SpillElem>(values: Vec<T>, dir: &Path, label: &str) -> anyhow::R
/// Size a freshly-created spill file to `byte_len` bytes, reserving the disk
/// blocks up front. We write the array through a mutable mmap, and writing to a
/// dirty mmap page the kernel can't back at writeback raises `SIGBUS` so a
/// dirty mmap page the kernel can't back at writeback raises `SIGBUS`, so a
/// plain sparse `set_len` would turn an out-of-space dev disk into a crash. On
/// Linux `posix_fallocate` allocates the blocks now, surfacing the shortfall as a
/// clean `ENOSPC` error here instead. Elsewhere we fall back to `set_len`.
@ -348,7 +348,7 @@ fn allocate_spill_file(file: &File, byte_len: usize, label: &str) -> anyhow::Res
/// Create a spill file in `dir` and immediately unlink it. On Unix the open
/// descriptor (and the mapping built from it) keep the inode alive, so the file is
/// invisible in the directory and its blocks are reclaimed automatically when the
/// map is dropped no cleanup, no leftovers across runs.
/// map is dropped: no cleanup, no leftovers across runs.
fn anon_file(dir: &Path, label: &str) -> anyhow::Result<File> {
std::fs::create_dir_all(dir)
.with_context(|| format!("creating spill directory {}", dir.display()))?;
@ -437,7 +437,7 @@ mod tests {
#[test]
fn spur_keys_survive_the_mmap_roundtrip() {
// Spur is a NonZeroU32 niche type exercises the SpillElem soundness claim.
// Spur is a NonZeroU32 niche type: exercises the SpillElem soundness claim.
let mut rodeo = lasso::Rodeo::default();
let keys: Vec<lasso::Spur> = (0..2000)
.map(|n| rodeo.get_or_intern(format!("token-{n}")))
@ -493,7 +493,7 @@ mod tests {
#[test]
fn builder_spurs_survive_the_mmap_roundtrip() {
// Spur is a NonZeroU32 niche type exercises the streamed write path for
// Spur is a NonZeroU32 niche type: exercises the streamed write path for
// the crime-records `location` column.
let mut rodeo = lasso::Rodeo::default();
let keys: Vec<lasso::Spur> = (0..3000)

View file

@ -315,7 +315,7 @@ fn parse_postcode_list(
Some(&pc_idx) if seen.insert(pc_idx) => {
entries.push((pc_idx, normalized));
}
Some(_) => {} // duplicate skip silently
Some(_) => {} // duplicate, skip silently
None => unknown.push(normalized),
}
}
@ -444,7 +444,7 @@ pub async fn get_export(
) -> Result<impl IntoResponse, axum::response::Response> {
let state = shared.load_state();
// Exporting requires an account no anonymous/demo exports.
// Exporting requires an account: no anonymous/demo exports.
if user.0.is_none() {
return Err((
StatusCode::UNAUTHORIZED,
@ -958,7 +958,7 @@ pub async fn get_export(
// Two sheets in both modes: "Selected" (just the features behind the
// active filters, including any POI amenity metrics) and "All Data"
// (every property/area feature, but no POI amenity counts/distances
// (every property/area feature, but no POI amenity counts/distances;
// those live only on the Selected sheet). The Selected sheet carries the
// dashboard link + screenshot only in bounds mode, where the export is
// tied to a map view; in list mode the user picked the postcodes
@ -1104,7 +1104,7 @@ pub async fn get_export(
// outcode summary row acts as the header for its postcodes.
sheet.group_symbols_above(true);
// Data rows one bold outcode summary row followed by its postcodes,
// Data rows: one bold outcode summary row followed by its postcodes,
// the latter wrapped in a collapsible outline group.
let data_start_row = desc_row + 1;
let mut row = data_start_row;