lgtm

server-rs/src/data.rs

@@ -1,6 +1,7 @@
 mod actual_listings;
 pub mod area_crime_averages;
 pub mod crime_by_year;
+pub mod crime_records;
 mod developments;
 mod places;
 mod poi;
@@ -65,6 +66,7 @@ where
 pub use actual_listings::{ActualListing, ActualListingData};
 pub use area_crime_averages::AreaCrimeAverages;
 pub use crime_by_year::CrimeByYearData;
+pub use crime_records::CrimeRecords;
 pub use developments::{DevelopmentData, DevelopmentSite};
 pub use places::{
     compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,

server-rs/src/data/area_crime_averages.rs

@@ -1,41 +1,61 @@
-//! Precomputed per-outcode and per-postcode-sector average crime rates.
+//! Precomputed per-outcode and per-postcode-sector average crime counts.
 //!
 //! The right pane shows each crime metric's national average (the global
 //! feature-histogram mean). To let users see how an area compares with its
-//! immediate surroundings, we also precompute the mean headline crime rate
-//! (`"X (avg/yr)"`) across every property in the selection's outcode (e.g.
-//! `"E14"`) and postcode sector (e.g. `"E14 2"`).
+//! immediate surroundings, we also show the mean average-annual crime count
+//! (`"X (/yr, 7y|2y)"`) across every property in the selection's outcode
+//! (e.g. `"E14"`) and postcode sector (e.g. `"E14 2"`).
 //!
-//! Crime figures are constant within a postcode (the pipeline merges them on
-//! the postcode key), so each postcode's value is read once — from its first
-//! row — and property-weighted by the postcode's row count. That keeps these
-//! averages on the same property-weighted basis as the national average, so the
+//! These averages are precomputed by the data pipeline
+//! (`pipeline/transform/area_crime_averages.py`) and loaded here from a side
+//! parquet. Crime figures are constant within a postcode, so the pipeline
+//! property-weights each postcode's value by its property count — keeping these
+//! averages on the same property-weighted basis as the per-selection mean, so the
 //! four numbers (this area / sector / outcode / nation) are directly comparable.
 
-use rustc_hash::FxHashMap;
+use std::path::Path;
 
-/// Crime-feature name suffix that marks an annualised headline-rate column
-/// (e.g. `"Burglary (avg/yr)"`). Stripped to derive the bare type name.
-pub const AVG_YR_SUFFIX: &str = " (avg/yr)";
+use anyhow::{bail, Context};
+use polars::prelude::PlRefPath;
+use polars::prelude::*;
+use rustc_hash::FxHashMap;
+use tracing::info;
+
+use super::run_polars_io;
+
+/// Marker that identifies an average-annual crime-count column (e.g.
+/// `"Burglary (/yr, 7y)"`). These are the filterable, area-comparable figures.
+/// The full column name is kept as the key, so each per-area mean aligns with the
+/// feature the frontend requests.
+pub const COUNT_MARKER: &str = " (/yr, ";
+
+/// `scope` column discriminator values written by the pipeline.
+const SCOPE_NATIONAL: &str = "national";
+const SCOPE_OUTCODE: &str = "outcode";
+const SCOPE_SECTOR: &str = "sector";
 
 pub struct AreaCrimeAverages {
-    /// Bare crime-type names (suffix stripped, e.g. `"Burglary"`), index-aligned
-    /// with the per-area mean vectors. Matches `CrimeYearStats.name`.
+    /// Full crime feature names (e.g. `"Burglary (/yr, 7y)"`), index-aligned with
+    /// the per-area mean vectors. Matches the feature names the frontend requests,
+    /// so each NumberLine can look its average up directly.
     pub crime_types: Vec<String>,
-    /// National mean headline rate per crime type (index-aligned with
+    /// National mean headline count per crime type (index-aligned with
     /// `crime_types`). An EXACT property-weighted mean over every postcode, so it
     /// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
     /// unlike the histogram-bin national average, which is biased upward for the
-    /// right-skewed crime densities. `NaN` where no postcode has data.
+    /// right-skewed crime counts. `NaN` where no postcode has data.
     pub national: Vec<f32>,
-    /// Outcode (e.g. `"E14"`) → mean headline rate per crime type. `NaN` where
+    /// Outcode (e.g. `"E14"`) → mean headline count per crime type. `NaN` where
     /// the outcode has no data for that type.
     pub by_outcode: FxHashMap<String, Vec<f32>>,
-    /// Postcode sector (e.g. `"E14 2"`) → mean headline rate per crime type.
+    /// Postcode sector (e.g. `"E14 2"`) → mean headline count per crime type.
     pub by_sector: FxHashMap<String, Vec<f32>>,
 }
 
 impl AreaCrimeAverages {
+    /// Empty table — used only by the test-only `AppState` builder (the real
+    /// server always loads the precomputed parquet).
+    #[cfg(test)]
     pub fn empty() -> Self {
         Self {
             crime_types: Vec::new(),
@@ -44,4 +64,202 @@ impl AreaCrimeAverages {
             by_sector: FxHashMap::default(),
         }
     }
+
+    pub fn load(path: &Path) -> anyhow::Result<Self> {
+        run_polars_io(|| Self::load_inner(path))
+    }
+
+    fn load_inner(path: &Path) -> anyhow::Result<Self> {
+        info!("Loading area crime averages from {}", path.display());
+        let pl_path = PlRefPath::try_from_path(path).with_context(|| {
+            format!(
+                "Failed to normalize area-crime-averages parquet path {}",
+                path.display()
+            )
+        })?;
+        let df = LazyFrame::scan_parquet(pl_path, Default::default())
+            .with_context(|| {
+                format!(
+                    "Failed to scan area-crime-averages parquet at {}",
+                    path.display()
+                )
+            })?
+            .collect()
+            .with_context(|| {
+                format!(
+                    "Failed to read area-crime-averages parquet at {}",
+                    path.display()
+                )
+            })?;
+
+        // Crime columns are those carrying the count marker; their full names
+        // are kept (in column order) as the keys, so every per-area mean vector is
+        // index-aligned with `crime_types` and matches the requested feature name.
+        let crime_cols: Vec<String> = df
+            .get_column_names()
+            .iter()
+            .filter(|name| name.contains(COUNT_MARKER))
+            .map(|name| name.to_string())
+            .collect();
+        if crime_cols.is_empty() {
+            bail!(
+                "area-crime-averages parquet at {} has no '*{COUNT_MARKER}*' count columns",
+                path.display()
+            );
+        }
+        let crime_types: Vec<String> = crime_cols.clone();
+        let n = crime_cols.len();
+
+        let scope_col = df
+            .column("scope")
+            .context("area-crime-averages parquet missing 'scope' column")?
+            .str()
+            .context("'scope' column is not a string")?;
+        let area_col = df
+            .column("area")
+            .context("area-crime-averages parquet missing 'area' column")?
+            .str()
+            .context("'area' column is not a string")?;
+
+        // Hold the casts alive while we borrow `Float32Chunked` views into them.
+        let casts: Vec<Column> = crime_cols
+            .iter()
+            .map(|name| {
+                df.column(name)
+                    .and_then(|col| col.cast(&DataType::Float32))
+                    .with_context(|| format!("Failed to read crime column '{name}' as f32"))
+            })
+            .collect::<anyhow::Result<Vec<_>>>()?;
+        let crime_views: Vec<&Float32Chunked> = casts
+            .iter()
+            .zip(crime_cols.iter())
+            .map(|(col, name)| {
+                col.f32()
+                    .with_context(|| format!("crime column '{name}' is not f32 after cast"))
+            })
+            .collect::<anyhow::Result<Vec<_>>>()?;
+
+        let read_values = |row: usize| -> Vec<f32> {
+            crime_views
+                .iter()
+                .map(|view| view.get(row).unwrap_or(f32::NAN))
+                .collect()
+        };
+
+        let mut national: Option<Vec<f32>> = None;
+        let mut by_outcode: FxHashMap<String, Vec<f32>> = FxHashMap::default();
+        let mut by_sector: FxHashMap<String, Vec<f32>> = FxHashMap::default();
+
+        for row in 0..df.height() {
+            let scope = scope_col
+                .get(row)
+                .with_context(|| format!("area-crime-averages row {row} has null scope"))?;
+            match scope {
+                SCOPE_NATIONAL => national = Some(read_values(row)),
+                SCOPE_OUTCODE => {
+                    if let Some(area) = area_col.get(row) {
+                        by_outcode.insert(area.to_string(), read_values(row));
+                    }
+                }
+                SCOPE_SECTOR => {
+                    if let Some(area) = area_col.get(row) {
+                        by_sector.insert(area.to_string(), read_values(row));
+                    }
+                }
+                other => bail!("area-crime-averages row {row} has unknown scope '{other}'"),
+            }
+        }
+
+        let national = national.context(
+            "area-crime-averages parquet has no 'national' row; regenerate it with \
+             pipeline.transform.area_crime_averages",
+        )?;
+        if national.len() != n {
+            bail!(
+                "area-crime-averages national row has {} values, expected {n}",
+                national.len()
+            );
+        }
+
+        info!(
+            outcodes = by_outcode.len(),
+            sectors = by_sector.len(),
+            crime_types = crime_types.len(),
+            "Area crime averages loaded"
+        );
+
+        Ok(Self {
+            crime_types,
+            national,
+            by_outcode,
+            by_sector,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn write_fixture(path: &Path) {
+        // national + one outcode (E14) + one sector (E14 2). Robbery is null for
+        // the outcode to exercise the NaN round-trip.
+        let mut df = df!(
+            "scope" => ["national", "outcode", "sector"],
+            "area" => ["", "E14", "E14 2"],
+            "Burglary (/yr, 7y)" => [8.75f32, 12.5, 12.5],
+            "Robbery (/yr, 7y)" => [Some(1.43f32), None, Some(2.0)],
+            // A non-crime column must be ignored by the loader.
+            "Median age" => [40.0f32, 41.0, 42.0],
+        )
+        .unwrap();
+        let mut file = std::fs::File::create(path).unwrap();
+        ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+    }
+
+    #[test]
+    fn load_round_trips_national_outcode_sector() {
+        let dir = std::env::temp_dir().join(format!("acavg-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("area_crime_averages.parquet");
+        write_fixture(&path);
+
+        let avgs = AreaCrimeAverages::load(&path).unwrap();
+
+        // Crime columns are discovered by the count marker; "Median age" is not.
+        assert_eq!(
+            avgs.crime_types,
+            vec!["Burglary (/yr, 7y)", "Robbery (/yr, 7y)"]
+        );
+        assert_eq!(avgs.national, vec![8.75, 1.43]);
+
+        let e14 = avgs.by_outcode.get("E14").unwrap();
+        assert_eq!(e14[0], 12.5);
+        // The null robbery value becomes NaN, which the consumer drops to None.
+        assert!(e14[1].is_nan());
+
+        let e14_2 = avgs.by_sector.get("E14 2").unwrap();
+        assert_eq!(e14_2, &vec![12.5, 2.0]);
+
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn load_rejects_parquet_without_national_row() {
+        let dir = std::env::temp_dir().join(format!("acavg-nonat-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("no_national.parquet");
+        let mut df = df!(
+            "scope" => ["outcode"],
+            "area" => ["E14"],
+            "Burglary (/yr, 7y)" => [12.5f32],
+        )
+        .unwrap();
+        let mut file = std::fs::File::create(&path).unwrap();
+        ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+
+        assert!(AreaCrimeAverages::load(&path).is_err());
+
+        std::fs::remove_dir_all(&dir).ok();
+    }
 }

server-rs/src/data/crime_records.rs

@@ -0,0 +1,534 @@
+//! Individual police.uk crime records (last 7 years) backing the right pane's
+//! "individual crimes" list and the `/api/crime-records` endpoint.
+//!
+//! This table is enormous — ~500M rows, because each incident is replicated to
+//! every postcode whose buffer covers it (see [`gather`](CrimeRecords::gather)),
+//! so it is NOT held as a `Vec<struct>`: each field is a flat columnar
+//! [`SpillVec`] (mmap-backed and kernel-reclaimable when `--spill-dir` is set),
+//! small string fields are dictionary-encoded, and the parquet is pre-sorted by
+//! postcode so each postcode's records are a contiguous `[start, start+count)`
+//! slice located via a CSR-style offset index. Resident RSS is ~0 until records
+//! are actually read.
+//!
+//! At ~500M rows the parquet's string columns (postcode/type/location/outcome)
+//! decode to tens of GB if read whole, so the loader never materialises the
+//! whole `DataFrame`: it streams the file in bounded row-count chunks (only the
+//! row groups overlapping each slice are decoded) and writes each column
+//! straight into its (optionally spilled) backing store via [`SpillVecBuilder`],
+//! keeping the transient footprint to one chunk plus the index maps.
+
+use std::fs::File;
+use std::path::Path;
+
+use anyhow::{bail, Context};
+use lasso::{Rodeo, RodeoReader, Spur};
+use polars::prelude::*;
+use rustc_hash::FxHashMap;
+use tracing::info;
+
+use super::run_polars_io;
+use super::spill::{SpillVec, SpillVecBuilder};
+
+/// Rows decoded per streaming slice. `with_slice` decodes only the row groups
+/// overlapping the slice, so the transient decode is roughly one chunk's worth
+/// (~tens of MB at the writer's ~123k-row groups) instead of the tens-of-GB
+/// whole-file `DataFrame`. The bound's only dependency on file layout is a
+/// reasonable input row-group size, which our pipeline writer produces.
+const CHUNK_ROWS: usize = 2_000_000;
+
+/// A resolved view of one record (strings dereferenced from the dictionaries).
+pub struct CrimeRecordView<'a> {
+    /// `year * 12 + (month - 1)`.
+    pub month_index: u32,
+    pub crime_type: &'a str,
+    pub outcome: Option<&'a str>,
+    pub location: Option<&'a str>,
+    pub lat: f32,
+    pub lon: f32,
+}
+
+pub struct CrimeRecords {
+    month: SpillVec<u32>,
+    ctype: SpillVec<u8>,
+    outcome: SpillVec<u8>,
+    location: SpillVec<Spur>,
+    lat: SpillVec<f32>,
+    lon: SpillVec<f32>,
+    /// Dictionary for `ctype` (bare crime type names, e.g. "Burglary").
+    crime_type_dict: Vec<String>,
+    /// Dictionary for `outcome`; index 0 is the empty/unknown sentinel.
+    outcome_dict: Vec<String>,
+    /// Resolver for the interned `location` strings (`""` means withheld).
+    location_resolver: RodeoReader,
+    /// Postcode → `(start, count)` into the columnar arrays (records for a
+    /// postcode are contiguous because the parquet is sorted by postcode).
+    by_postcode: FxHashMap<String, (u32, u32)>,
+}
+
+impl CrimeRecords {
+    #[cfg(test)]
+    pub fn empty() -> Self {
+        Self {
+            month: SpillVec::owned(Vec::new()),
+            ctype: SpillVec::owned(Vec::new()),
+            outcome: SpillVec::owned(Vec::new()),
+            location: SpillVec::owned(Vec::new()),
+            lat: SpillVec::owned(Vec::new()),
+            lon: SpillVec::owned(Vec::new()),
+            crime_type_dict: Vec::new(),
+            outcome_dict: vec![String::new()],
+            location_resolver: Rodeo::default().into_reader(),
+            by_postcode: FxHashMap::default(),
+        }
+    }
+
+    /// Number of records stored for a postcode (0 if none).
+    pub fn total_for(&self, postcode: &str) -> u32 {
+        self.by_postcode.get(postcode).map_or(0, |&(_, c)| c)
+    }
+
+    /// Resolve a record index to a borrowing view.
+    pub fn view(&self, idx: u32) -> CrimeRecordView<'_> {
+        let i = idx as usize;
+        let outcome_idx = self.outcome[i] as usize;
+        let outcome = self
+            .outcome_dict
+            .get(outcome_idx)
+            .filter(|s| !s.is_empty())
+            .map(String::as_str);
+        let location = self.location_resolver.resolve(&self.location[i]);
+        CrimeRecordView {
+            month_index: self.month[i],
+            crime_type: self
+                .crime_type_dict
+                .get(self.ctype[i] as usize)
+                .map_or("", String::as_str),
+            outcome,
+            location: (!location.is_empty()).then_some(location),
+            lat: self.lat[i],
+            lon: self.lon[i],
+        }
+    }
+
+    /// Record indices across `postcodes`, newest first, optionally restricted to
+    /// months `>= since_month`. These are exactly the incidents counted for the
+    /// selected postcodes — for a single postcode that is its precise incident
+    /// list; for a multi-postcode selection a boundary incident counted for
+    /// several postcodes appears once per postcode, matching the count. We do not
+    /// de-duplicate because police.uk snaps many genuinely distinct incidents
+    /// (especially anti-social behaviour) to the same point/month and provides no
+    /// per-incident id to tell a true duplicate from two real incidents apart.
+    pub fn gather(&self, postcodes: &[&str], since_month: Option<u32>) -> Vec<u32> {
+        let month = self.month.as_slice();
+        let mut out: Vec<u32> = Vec::new();
+        for pc in postcodes {
+            let Some(&(start, count)) = self.by_postcode.get(*pc) else {
+                continue;
+            };
+            for i in start..start + count {
+                if since_month.map_or(true, |s| month[i as usize] >= s) {
+                    out.push(i);
+                }
+            }
+        }
+        out.sort_unstable_by(|&a, &b| month[b as usize].cmp(&month[a as usize]));
+        out
+    }
+
+    pub fn load(path: &Path, spill_dir: Option<&Path>) -> anyhow::Result<Self> {
+        run_polars_io(|| Self::load_inner(path, spill_dir, CHUNK_ROWS))
+    }
+
+    fn load_inner(
+        path: &Path,
+        spill_dir: Option<&Path>,
+        chunk_rows: usize,
+    ) -> anyhow::Result<Self> {
+        // A zero chunk size would loop forever; the public entry point passes the
+        // const, but tests parameterise this to exercise chunk boundaries.
+        let chunk_rows = chunk_rows.max(1);
+        info!("Loading crime records from {}", path.display());
+
+        // Read the footer once for the row count, and keep it to hand to every
+        // per-chunk reader so the 2.9MB metadata is never re-parsed.
+        let metadata = {
+            let file = File::open(path).with_context(|| {
+                format!("Failed to open crime-records parquet at {}", path.display())
+            })?;
+            ParquetReader::new(file)
+                .get_metadata()
+                .with_context(|| {
+                    format!("Failed to read crime-records parquet metadata at {}", path.display())
+                })?
+                .clone()
+        };
+        let n = metadata.num_rows;
+        // Record indices are stored as `u32` (and `by_postcode` holds `(start,
+        // count)` as `u32`), so the table must fit in that index space.
+        if n > u32::MAX as usize {
+            bail!("crime-records parquet has {n} rows, exceeding the u32 record-index limit");
+        }
+
+        // Columns that, when spilling, are written straight into mmap-backed files
+        // as we stream — so the ~9GB of columnar data never lands on the heap.
+        let mut month = SpillVecBuilder::<u32>::with_len(n, spill_dir, "crime_month")?;
+        let mut ctype = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_ctype")?;
+        let mut outcome = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_outcome")?;
+        let mut location = SpillVecBuilder::<Spur>::with_len(n, spill_dir, "crime_location")?;
+        let mut lat = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lat")?;
+        let mut lon = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lon")?;
+
+        let mut crime_type_dict: Vec<String> = Vec::new();
+        let mut type_index: FxHashMap<String, u8> = FxHashMap::default();
+        // Outcome index 0 is the empty/unknown sentinel.
+        let mut outcome_dict: Vec<String> = vec![String::new()];
+        let mut outcome_index: FxHashMap<String, u8> = FxHashMap::default();
+        let mut rodeo = Rodeo::default();
+        let empty_spur = rodeo.get_or_intern("");
+
+        let mut by_postcode: FxHashMap<String, (u32, u32)> = FxHashMap::default();
+        let mut cur_pc: Option<String> = None;
+        let mut cur_start: u32 = 0;
+        // Absolute record index across all chunks; drives both the CSR index and
+        // the column builders' write order.
+        let mut global_row: u32 = 0;
+
+        let columns: Vec<String> = ["postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+
+        let mut offset = 0usize;
+        while offset < n {
+            let len = chunk_rows.min(n - offset);
+            let file = File::open(path).with_context(|| {
+                format!("Failed to open crime-records parquet at {}", path.display())
+            })?;
+            let mut reader = ParquetReader::new(file);
+            reader.set_metadata(metadata.clone());
+            // `with_slice` only decodes the row groups overlapping `[offset,
+            // offset+len)` (the file is memory-mapped, so untouched groups are
+            // never faulted in), capping the transient decode to one chunk.
+            let df = reader
+                .with_columns(Some(columns.clone()))
+                .with_slice(Some((offset, len)))
+                .finish()
+                .with_context(|| {
+                    format!(
+                        "Failed to read crime-records rows [{offset}, {}) from {}",
+                        offset + len,
+                        path.display()
+                    )
+                })?;
+
+            let postcode_col = df
+                .column("postcode")
+                .context("crime-records parquet missing 'postcode'")?
+                .str()
+                .context("'postcode' is not a string")?;
+            let month_col = df
+                .column("month_index")
+                .context("crime-records parquet missing 'month_index'")?
+                .cast(&DataType::Int32)
+                .context("'month_index' not castable to i32")?;
+            let month_ca = month_col.i32().context("'month_index' is not i32")?;
+            // Months are `year*12 + month0` (~24_000), always positive. A null or
+            // non-positive value means a corrupt parquet; fail loudly rather than
+            // silently clamping it to 0 and later rendering it as "0000-01".
+            if month_ca.null_count() > 0 {
+                bail!("crime-records 'month_index' has null values (corrupt parquet)");
+            }
+            match month_ca.min() {
+                Some(m) if m > 0 => {}
+                _ => bail!("crime-records 'month_index' must be a positive year*12+month index"),
+            }
+            let type_col = df
+                .column("crime_type")
+                .context("crime-records parquet missing 'crime_type'")?
+                .str()
+                .context("'crime_type' is not a string")?;
+            let location_col = df
+                .column("location")
+                .context("crime-records parquet missing 'location'")?
+                .str()
+                .context("'location' is not a string")?;
+            let outcome_col = df
+                .column("outcome")
+                .context("crime-records parquet missing 'outcome'")?
+                .str()
+                .context("'outcome' is not a string")?;
+            let lat_col = df
+                .column("lat")
+                .context("crime-records parquet missing 'lat'")?
+                .cast(&DataType::Float32)?;
+            let lat_ca = lat_col.f32().context("'lat' is not f32")?;
+            let lon_col = df
+                .column("lon")
+                .context("crime-records parquet missing 'lon'")?
+                .cast(&DataType::Float32)?;
+            let lon_ca = lon_col.f32().context("'lon' is not f32")?;
+
+            let height = df.height();
+            for row in 0..height {
+                // CSR index: the parquet is sorted by postcode, so a change in the
+                // postcode value (across chunk boundaries too) closes the previous
+                // run and opens a new one.
+                let pc = postcode_col
+                    .get(row)
+                    .with_context(|| {
+                        format!("crime-records row {} has null postcode", offset + row)
+                    })?
+                    .trim();
+                if cur_pc.as_deref() != Some(pc) {
+                    if let Some(prev) = cur_pc.take() {
+                        by_postcode.insert(prev, (cur_start, global_row - cur_start));
+                    }
+                    cur_pc = Some(pc.to_string());
+                    cur_start = global_row;
+                }
+
+                month.push(month_ca.get(row).unwrap() as u32);
+
+                let ty = type_col.get(row).unwrap_or("");
+                let ty_id = match type_index.get(ty) {
+                    Some(&id) => id,
+                    None => {
+                        let id = u8::try_from(crime_type_dict.len())
+                            .context("more than 256 distinct crime types")?;
+                        crime_type_dict.push(ty.to_string());
+                        type_index.insert(ty.to_string(), id);
+                        id
+                    }
+                };
+                ctype.push(ty_id);
+
+                let oc = outcome_col.get(row).unwrap_or("");
+                let oc_id = if oc.is_empty() {
+                    0
+                } else {
+                    match outcome_index.get(oc) {
+                        Some(&id) => id,
+                        None => {
+                            let id = u8::try_from(outcome_dict.len())
+                                .context("more than 256 distinct outcomes")?;
+                            outcome_dict.push(oc.to_string());
+                            outcome_index.insert(oc.to_string(), id);
+                            id
+                        }
+                    }
+                };
+                outcome.push(oc_id);
+
+                let loc = location_col.get(row).unwrap_or("");
+                location.push(if loc.is_empty() {
+                    empty_spur
+                } else {
+                    rodeo.get_or_intern(loc)
+                });
+
+                lat.push(lat_ca.get(row).unwrap_or(f32::NAN));
+                lon.push(lon_ca.get(row).unwrap_or(f32::NAN));
+
+                global_row += 1;
+            }
+
+            offset += len;
+        }
+        if let Some(prev) = cur_pc.take() {
+            by_postcode.insert(prev, (cur_start, global_row - cur_start));
+        }
+        debug_assert_eq!(global_row as usize, n, "streamed fewer rows than the parquet declares");
+
+        let records = Self {
+            month: month.finish()?,
+            ctype: ctype.finish()?,
+            outcome: outcome.finish()?,
+            location: location.finish()?,
+            lat: lat.finish()?,
+            lon: lon.finish()?,
+            crime_type_dict,
+            outcome_dict,
+            location_resolver: rodeo.into_reader(),
+            by_postcode,
+        };
+
+        info!(
+            records = n,
+            postcodes = records.by_postcode.len(),
+            crime_types = records.crime_type_dict.len(),
+            outcomes = records.outcome_dict.len(),
+            "Crime records loaded"
+        );
+        Ok(records)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn write_fixture(path: &Path) {
+        // Two postcodes, postcode-sorted. AA1 1AA has 3 records across two
+        // months (a null outcome and a null location), BB2 2BB has 1.
+        let mut df = df!(
+            "postcode" => ["AA1 1AA", "AA1 1AA", "AA1 1AA", "BB2 2BB"],
+            "month_index" => [24300i32, 24300, 24290, 24305],
+            "crime_type" => ["Burglary", "Burglary", "Robbery", "Drugs"],
+            "location" => [Some("On or near A St"), Some("On or near A St"), None, Some("On or near B Rd")],
+            "outcome" => [Some("Under investigation"), None, Some("Court result"), None],
+            "lat" => [51.5f32, 51.5, 51.6, 52.0],
+            "lon" => [-0.1f32, -0.1, -0.2, -1.0],
+        )
+        .unwrap();
+        let mut file = std::fs::File::create(path).unwrap();
+        ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+    }
+
+    #[test]
+    fn loads_indexes_and_gathers() {
+        let dir = std::env::temp_dir().join(format!("crimerec-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("records.parquet");
+        write_fixture(&path);
+
+        let recs = CrimeRecords::load(&path, None).unwrap();
+        assert_eq!(recs.total_for("AA1 1AA"), 3);
+        assert_eq!(recs.total_for("BB2 2BB"), 1);
+        assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
+
+        // Newest-first across the two postcodes.
+        let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
+        assert_eq!(all.len(), 4);
+        let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
+        assert_eq!(months, vec![24305, 24300, 24300, 24290]);
+
+        // `since` window filter (keep months >= 24300).
+        assert_eq!(recs.gather(&["AA1 1AA"], Some(24300)).len(), 2);
+
+        // String resolution + null handling.
+        let robbery = all
+            .iter()
+            .map(|&i| recs.view(i))
+            .find(|v| v.crime_type == "Robbery")
+            .unwrap();
+        assert_eq!(robbery.outcome, Some("Court result"));
+        assert_eq!(robbery.location, None); // null location → None
+        // Two records have a null outcome (an AA1 Burglary and the BB2 Drugs).
+        let null_outcomes = all
+            .iter()
+            .map(|&i| recs.view(i))
+            .filter(|v| v.outcome.is_none())
+            .count();
+        assert_eq!(null_outcomes, 2);
+
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    /// The CSR per-postcode index and the column builders must compose correctly
+    /// across streaming chunk boundaries — including a postcode run split between
+    /// two chunks. Forces `chunk_rows = 2` over the 4-row fixture so AA1 1AA's
+    /// three records straddle the boundary (rows 0,1 in chunk 0; row 2 in chunk 1)
+    /// and is exercised both heap-backed (no spill) and mmap-backed (spill).
+    #[test]
+    fn streams_across_chunk_boundaries() {
+        let base = std::env::temp_dir().join(format!("crimerec-chunk-{}", std::process::id()));
+        std::fs::create_dir_all(&base).unwrap();
+        let path = base.join("records.parquet");
+        write_fixture(&path);
+        let spill = base.join("spill");
+        std::fs::create_dir_all(&spill).unwrap();
+
+        for spill_dir in [None, Some(spill.as_path())] {
+            let recs = CrimeRecords::load_inner(&path, spill_dir, 2).unwrap();
+            // Counts match regardless of how the runs were split across chunks.
+            assert_eq!(recs.total_for("AA1 1AA"), 3);
+            assert_eq!(recs.total_for("BB2 2BB"), 1);
+            assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
+
+            // Full gather, newest-first, identical to the single-chunk load.
+            let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
+            assert_eq!(all.len(), 4);
+            let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
+            assert_eq!(months, vec![24305, 24300, 24300, 24290]);
+
+            // The run that straddled the boundary still resolves its strings.
+            let robbery = all
+                .iter()
+                .map(|&i| recs.view(i))
+                .find(|v| v.crime_type == "Robbery")
+                .unwrap();
+            assert_eq!(robbery.outcome, Some("Court result"));
+            assert_eq!(robbery.location, None);
+        }
+
+        std::fs::remove_dir_all(&base).ok();
+    }
+
+    /// Peak/resident RSS in MiB from `/proc/self/status` (Linux only).
+    fn rss_mib() -> (f64, f64) {
+        let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
+        let field = |key: &str| -> f64 {
+            status
+                .lines()
+                .find(|l| l.starts_with(key))
+                .and_then(|l| l.split_whitespace().nth(1))
+                .and_then(|kb| kb.parse::<f64>().ok())
+                .map_or(0.0, |kb| kb / 1024.0)
+        };
+        (field("VmHWM:"), field("VmRSS:"))
+    }
+
+    /// Manual, real-data smoke test: load the actual ~500M-row parquet and report
+    /// peak RSS, proving the streaming + spill load completes without the
+    /// tens-of-GB `DataFrame` materialisation that OOMed the old `.collect()`.
+    ///
+    /// Run with:
+    ///   PPC_REAL_CRIME_RECORDS=/path/to/crime_records.parquet \
+    ///     cargo test --bins -- --ignored --nocapture real_crime_records_load_is_bounded
+    #[test]
+    #[ignore = "needs the full crime_records.parquet; run manually"]
+    fn real_crime_records_load_is_bounded() {
+        let path = std::env::var("PPC_REAL_CRIME_RECORDS")
+            .unwrap_or_else(|_| "../property-data/crime_records.parquet".to_string());
+        let path = Path::new(&path);
+        if !path.exists() {
+            eprintln!("skipping: {} not found", path.display());
+            return;
+        }
+        let spill = std::env::var("PPC_REAL_SPILL")
+            .unwrap_or_else(|_| "../.tmp/crime-spill-realtest".to_string());
+        let spill = Path::new(&spill);
+        std::fs::create_dir_all(spill).unwrap();
+
+        let (hwm_before, _rss_before) = rss_mib();
+        let start = std::time::Instant::now();
+        let recs = CrimeRecords::load(path, Some(spill)).unwrap();
+        let elapsed = start.elapsed();
+        let (hwm_after, rss_after) = rss_mib();
+
+        let total: u64 = recs.by_postcode.values().map(|&(_, c)| c as u64).sum();
+        eprintln!(
+            "loaded {} records across {} postcodes in {:.1}s | RSS peak {:.0}->{:.0} MiB (Δ{:.0}) resident now {:.0} MiB",
+            total,
+            recs.by_postcode.len(),
+            elapsed.as_secs_f64(),
+            hwm_before,
+            hwm_after,
+            hwm_after - hwm_before,
+            rss_after,
+        );
+
+        assert!(recs.by_postcode.len() > 0, "expected at least one postcode");
+        assert!(total > 0, "expected at least one record");
+        // The old `.collect()` decoded all rows' string columns at once (tens of
+        // GB). Streaming must keep the peak growth far below that; a generous 20GiB
+        // ceiling still proves we never materialise the whole file.
+        assert!(
+            hwm_after - hwm_before < 20_480.0,
+            "peak RSS grew by {:.0} MiB during load — streaming/spill not bounding memory",
+            hwm_after - hwm_before
+        );
+
+        std::fs::remove_dir_all(spill).ok();
+    }
+}

server-rs/src/data/property/mod.rs

@@ -26,9 +26,7 @@ use rustc_hash::FxHashMap;
 use serde::Serialize;
 
 use crate::consts::NAN_U16;
-use crate::data::area_crime_averages::{AreaCrimeAverages, AVG_YR_SUFFIX};
 use crate::data::spill::SpillVec;
-use crate::utils::{postcode_outcode, postcode_sector};
 
 #[derive(Serialize, Clone)]
 pub struct RenovationEvent {
@@ -226,109 +224,6 @@ impl PropertyData {
             num_numeric: self.num_numeric,
         }
     }
-
-    /// Precompute mean headline crime rates nationally and per outcode / postcode
-    /// sector.
-    ///
-    /// Crime values are identical for every property in a postcode (the pipeline
-    /// merges them on the postcode key), so each postcode is sampled once from
-    /// its first row and property-weighted by its row count. All three scopes use
-    /// the same exact property-weighted estimator over the same row universe as
-    /// the per-selection mean, so the four numbers shown in a crime row (this
-    /// selection / sector / outcode / nation) are directly comparable — without
-    /// the upward bias of the histogram-bin national average.
-    pub fn compute_area_crime_averages(&self) -> AreaCrimeAverages {
-        // Crime headline columns are exactly the " (avg/yr)" features.
-        let crime_indices: Vec<usize> = self
-            .feature_names
-            .iter()
-            .enumerate()
-            .filter(|(_, name)| name.ends_with(AVG_YR_SUFFIX))
-            .map(|(idx, _)| idx)
-            .collect();
-        if crime_indices.is_empty() {
-            return AreaCrimeAverages::empty();
-        }
-        let crime_types: Vec<String> = crime_indices
-            .iter()
-            .map(|&idx| {
-                self.feature_names[idx]
-                    .strip_suffix(AVG_YR_SUFFIX)
-                    .unwrap_or(&self.feature_names[idx])
-                    .to_string()
-            })
-            .collect();
-        let n = crime_indices.len();
-
-        // (weighted value sum, weight) accumulators per crime type.
-        let mut nat_sums = vec![0.0f64; n];
-        let mut nat_weights = vec![0u64; n];
-        let mut out_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
-        let mut sec_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
-
-        for (key, rows) in &self.postcode_row_index {
-            let Some(&first) = rows.first() else { continue };
-            let count = rows.len() as u64;
-            let postcode = self.postcode_interner.resolve(key);
-            let outcode = postcode_outcode(postcode);
-            let sector = postcode_sector(postcode);
-
-            for (j, &fi) in crime_indices.iter().enumerate() {
-                // A NaN value is "no crime data for this postcode" — skip it so
-                // it dilutes neither the sum nor the weight (a genuine gap, not
-                // a zero), exactly as the global histogram excludes it.
-                let value = self.get_feature(first as usize, fi);
-                if !value.is_finite() {
-                    continue;
-                }
-                let weighted = value as f64 * count as f64;
-                // National counts every postcode (the population the global mean
-                // is built over); outcode/sector only when the postcode parses.
-                nat_sums[j] += weighted;
-                nat_weights[j] += count;
-                if let Some(outcode) = outcode {
-                    let acc = out_acc
-                        .entry(outcode.to_string())
-                        .or_insert_with(|| (vec![0.0; n], vec![0; n]));
-                    acc.0[j] += weighted;
-                    acc.1[j] += count;
-                }
-                if let Some(sector) = sector {
-                    let acc = sec_acc
-                        .entry(sector.to_string())
-                        .or_insert_with(|| (vec![0.0; n], vec![0; n]));
-                    acc.0[j] += weighted;
-                    acc.1[j] += count;
-                }
-            }
-        }
-
-        let means_of = |sums: &[f64], weights: &[u64]| -> Vec<f32> {
-            sums.iter()
-                .zip(weights.iter())
-                .map(|(&sum, &weight)| {
-                    if weight == 0 {
-                        f32::NAN
-                    } else {
-                        (sum / weight as f64) as f32
-                    }
-                })
-                .collect()
-        };
-        let finalize =
-            |acc: FxHashMap<String, (Vec<f64>, Vec<u64>)>| -> FxHashMap<String, Vec<f32>> {
-                acc.into_iter()
-                    .map(|(area, (sums, weights))| (area, means_of(&sums, &weights)))
-                    .collect()
-            };
-
-        AreaCrimeAverages {
-            crime_types,
-            national: means_of(&nat_sums, &nat_weights),
-            by_outcode: finalize(out_acc),
-            by_sector: finalize(sec_acc),
-        }
-    }
 }
 
 #[cfg(test)]

server-rs/src/data/spill.rs

@@ -133,6 +133,138 @@ impl SpillVec<u16> {
     }
 }
 
+/// Builds a [`SpillVec<T>`] incrementally by `push`ing elements one at a time,
+/// when the final length is known up front but the values arrive in a stream.
+///
+/// When a spill `dir` is set (and the length is non-zero) the backing store is a
+/// pre-sized, memory-mapped file and each pushed element is written straight into
+/// it — so the array never exists as a heap `Vec` and is never copied a second
+/// time on finalisation, unlike [`SpillVec::maybe_spill`], which takes an
+/// already-built `Vec` and so needs the whole thing resident first. Without a
+/// spill dir it accumulates into an owned `Vec` (production behaviour, identical
+/// resident cost to the old `Vec::with_capacity` + `maybe_spill`). This is what
+/// lets the ~500M-row crime-records columns load without a multi-GB heap spike.
+///
+/// Exactly `len` elements must be pushed: pushing more panics, and finishing with
+/// fewer is an error.
+pub struct SpillVecBuilder<T: SpillElem> {
+    backing: Builder<T>,
+}
+
+enum Builder<T: SpillElem> {
+    Owned { values: Vec<T>, len: usize },
+    Mapped {
+        map: MmapMut,
+        len: usize,
+        cursor: usize,
+        label: &'static str,
+        _marker: PhantomData<T>,
+    },
+}
+
+impl<T: SpillElem> SpillVecBuilder<T> {
+    /// Create a builder for exactly `len` elements. Spills to `dir` when it is set
+    /// and `len > 0` (a zero-length mmap is invalid); otherwise reserves an owned
+    /// `Vec`. `label` names the backing file for diagnostics.
+    pub fn with_len(len: usize, dir: Option<&Path>, label: &'static str) -> anyhow::Result<Self> {
+        match dir {
+            Some(dir) if len > 0 => {
+                let byte_len = len * std::mem::size_of::<T>();
+                let file = anon_file(dir, label)?;
+                allocate_spill_file(&file, byte_len, label)?;
+                // SAFETY: `file` is a freshly-created, exclusively-owned regular
+                // file sized to exactly `byte_len`; no other mapping aliases it.
+                let map = unsafe { MmapMut::map_mut(&file) }.with_context(|| {
+                    format!("mapping spill file for '{label}' ({byte_len} bytes)")
+                })?;
+                Ok(Self {
+                    backing: Builder::Mapped {
+                        map,
+                        len,
+                        cursor: 0,
+                        label,
+                        _marker: PhantomData,
+                    },
+                })
+            }
+            _ => Ok(Self {
+                backing: Builder::Owned {
+                    values: Vec::with_capacity(len),
+                    len,
+                },
+            }),
+        }
+    }
+
+    /// Append one element. Panics if more than the declared `len` elements are
+    /// pushed — enforced identically on both backings so a streaming bug fails
+    /// the same way in production (owned) as in dev (spilled).
+    #[inline]
+    pub fn push(&mut self, value: T) {
+        match &mut self.backing {
+            Builder::Owned { values, len } => {
+                assert!(
+                    values.len() < *len,
+                    "SpillVecBuilder overflow: pushed more than {len} elements"
+                );
+                values.push(value);
+            }
+            Builder::Mapped {
+                map, len, cursor, ..
+            } => {
+                assert!(
+                    *cursor < *len,
+                    "SpillVecBuilder overflow: pushed more than {len} elements"
+                );
+                // SAFETY: an mmap base is page-aligned (hence aligned for `T`); the
+                // mapping holds exactly `len * size_of::<T>()` bytes and `cursor <
+                // len`, so this writes one in-bounds, aligned `T`. `T: SpillElem`
+                // is `Copy` and padding-free, so the stored byte image is fully
+                // defined and reads back as the same value.
+                unsafe { map.as_mut_ptr().cast::<T>().add(*cursor).write(value) };
+                *cursor += 1;
+            }
+        }
+    }
+
+    /// Seal the builder into a read-only [`SpillVec`]. Exactly the declared `len`
+    /// elements must have been pushed, on both backings.
+    pub fn finish(self) -> anyhow::Result<SpillVec<T>> {
+        match self.backing {
+            Builder::Owned { values, len } => {
+                if values.len() != len {
+                    anyhow::bail!(
+                        "spill builder finished with {} of {len} elements written",
+                        values.len()
+                    );
+                }
+                Ok(SpillVec::Owned(values))
+            }
+            Builder::Mapped {
+                map,
+                len,
+                cursor,
+                label,
+                ..
+            } => {
+                if cursor != len {
+                    anyhow::bail!(
+                        "spill builder for '{label}' finished with {cursor} of {len} elements written"
+                    );
+                }
+                let map = map
+                    .make_read_only()
+                    .with_context(|| format!("sealing spill file for '{label}'"))?;
+                Ok(SpillVec::Mapped {
+                    map,
+                    len,
+                    _marker: PhantomData,
+                })
+            }
+        }
+    }
+}
+
 impl<T: SpillElem> std::ops::Deref for SpillVec<T> {
     type Target = [T];
 
@@ -327,4 +459,105 @@ mod tests {
         assert!(matches!(mapped, SpillVec::Owned(_)));
         assert!(mapped.is_empty());
     }
+
+    #[test]
+    fn builder_streams_identically_owned_and_mapped() {
+        let values: Vec<u32> = (0..50_000u32).map(|n| n.wrapping_mul(2_246_822_519)).collect();
+
+        // Owned path (no spill dir): pushes accumulate into a heap Vec.
+        let mut owned = SpillVecBuilder::<u32>::with_len(values.len(), None, "u32_owned").unwrap();
+        for &v in &values {
+            owned.push(v);
+        }
+        let owned = owned.finish().unwrap();
+        assert!(matches!(owned, SpillVec::Owned(_)));
+        assert_eq!(&*owned, values.as_slice());
+
+        // Mapped path (spill dir): pushes write straight into the mmap, no heap copy.
+        let dir = TempDir::new("builder-u32");
+        let mut mapped =
+            SpillVecBuilder::<u32>::with_len(values.len(), Some(dir.path()), "u32_mapped").unwrap();
+        for &v in &values {
+            mapped.push(v);
+        }
+        let mapped = mapped.finish().unwrap();
+        assert!(matches!(mapped, SpillVec::Mapped { .. }));
+        // The mmap-backed slice must be byte-identical to the streamed input.
+        assert_eq!(&*mapped, values.as_slice());
+    }
+
+    #[test]
+    fn builder_spurs_survive_the_mmap_roundtrip() {
+        // Spur is a NonZeroU32 niche type — exercises the streamed write path for
+        // the crime-records `location` column.
+        let mut rodeo = lasso::Rodeo::default();
+        let keys: Vec<lasso::Spur> = (0..3000)
+            .map(|n| rodeo.get_or_intern(format!("loc-{n}")))
+            .collect();
+
+        let dir = TempDir::new("builder-spur");
+        let mut builder =
+            SpillVecBuilder::<lasso::Spur>::with_len(keys.len(), Some(dir.path()), "spurs").unwrap();
+        for &k in &keys {
+            builder.push(k);
+        }
+        let mapped = builder.finish().unwrap();
+        assert!(matches!(mapped, SpillVec::Mapped { .. }));
+        assert_eq!(&*mapped, keys.as_slice());
+        let reader = rodeo.into_resolver();
+        for (idx, &key) in mapped.iter().enumerate() {
+            assert_eq!(reader.resolve(&key), format!("loc-{idx}"));
+        }
+    }
+
+    #[test]
+    fn builder_zero_len_stays_owned() {
+        let dir = TempDir::new("builder-empty");
+        let builder = SpillVecBuilder::<u32>::with_len(0, Some(dir.path()), "empty").unwrap();
+        let v = builder.finish().unwrap();
+        // A zero-length mmap is invalid, so empties fall back to an owned Vec.
+        assert!(matches!(v, SpillVec::Owned(_)));
+        assert!(v.is_empty());
+    }
+
+    #[test]
+    fn builder_underfill_is_an_error() {
+        let dir = TempDir::new("builder-underfill");
+        let mut builder =
+            SpillVecBuilder::<u32>::with_len(4, Some(dir.path()), "underfill").unwrap();
+        builder.push(1);
+        builder.push(2);
+        // Sealing a spilling builder before all declared elements are written fails
+        // rather than exposing uninitialised mmap tail bytes as valid data.
+        assert!(builder.finish().is_err());
+    }
+
+    #[test]
+    #[should_panic(expected = "overflow")]
+    fn builder_overfill_panics() {
+        let dir = TempDir::new("builder-overfill");
+        let mut builder = SpillVecBuilder::<u32>::with_len(2, Some(dir.path()), "overfill").unwrap();
+        builder.push(1);
+        builder.push(2);
+        builder.push(3); // one past the declared length
+    }
+
+    #[test]
+    fn builder_owned_underfill_is_an_error() {
+        // The owned (no-spill, production) path enforces the declared length too,
+        // so a streaming bug can't silently yield a short array in release builds.
+        let mut builder = SpillVecBuilder::<u32>::with_len(4, None, "owned-underfill").unwrap();
+        builder.push(1);
+        builder.push(2);
+        assert!(builder.finish().is_err());
+    }
+
+    #[test]
+    #[should_panic(expected = "overflow")]
+    fn builder_owned_overfill_panics() {
+        let mut builder = SpillVecBuilder::<u32>::with_len(2, None, "owned-overfill").unwrap();
+        builder.push(1);
+        builder.push(2);
+        builder.push(3); // one past the declared length
+    }
 }

server-rs/src/features.rs

@@ -68,6 +68,59 @@ pub struct FeatureGroup {
     pub features: &'static [Feature],
 }
 
+/// Expand each crime type into its two filterable features: a 7-year and a
+/// 2-year window. Each is the average number of recorded incidents per year (the
+/// raw, absolute count — no per-area or per-capita normalisation). The names must
+/// match the `"{type} (/yr, 7y|2y)"` columns written by `crime_spatial`. The
+/// per-incident records are NOT a feature (they are a display-only side table the
+/// server loads directly), so they never appear here and are not filterable.
+macro_rules! crime_features {
+    ($( ($base:literal, $blurb:literal) ),+ $(,)?) => {
+        &[ $(
+            Feature::Numeric(FeatureConfig {
+                name: concat!($base, " (/yr, 7y)"),
+                bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
+                step: 0.1,
+                description: concat!($blurb, " — average recorded incidents per year (last 7 years)"),
+                detail: concat!(
+                    $blurb,
+                    ", as the average number of recorded incidents per year, over the last \
+                     7 years. Counted from police.uk street-level crime points (anonymised, \
+                     snapped to nearby map points) that fall near the postcode boundary — \
+                     the raw, absolute count, with no per-area or per-capita adjustment. \
+                     Computed over the months the local police force actually published; \
+                     known force gaps (e.g. Greater Manchester since mid-2019) are excluded, \
+                     not counted as zero crime."
+                ),
+                source: "crime",
+                prefix: "",
+                suffix: "",
+                raw: false,
+                absolute: false,
+            }),
+            Feature::Numeric(FeatureConfig {
+                name: concat!($base, " (/yr, 2y)"),
+                bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
+                step: 0.1,
+                description: concat!($blurb, " — average recorded incidents per year (last 2 years)"),
+                detail: concat!(
+                    $blurb,
+                    ", as the average number of recorded incidents per year, over the last \
+                     2 years — a more recent but noisier window than the 7-year figure. From \
+                     police.uk street-level crime points near the postcode boundary (the raw, \
+                     absolute count), over the months the local force published (gaps \
+                     excluded, not zeroed)."
+                ),
+                source: "crime",
+                prefix: "",
+                suffix: "",
+                raw: false,
+                absolute: false,
+            }),
+        )+ ]
+    };
+}
+
 pub static FEATURE_GROUPS: &[FeatureGroup] = &[
     FeatureGroup {
         name: "Properties",
@@ -472,247 +525,41 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
     },
     FeatureGroup {
         name: "Crime",
-        features: &[
-            Feature::Numeric(FeatureConfig {
-                name: "Serious crime (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Relative density of serious crime categories near the postcode",
-                detail: "Combined density of violence, robbery, burglary, and weapons possession near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Minor crime (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Relative density of minor crime categories near the postcode",
-                detail: "Combined density of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Violence and sexual offences (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly violent and sexual offences in the area",
-                detail: "Average number of violence and sexual offences per year near the postcode, from police.uk street-level crime data. Includes assault, harassment, and sexual offences.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Burglary (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly burglary offences in the area",
-                detail: "Average number of burglary offences per year near the postcode, from police.uk street-level crime data. Includes residential and commercial burglary.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Robbery (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly robbery offences in the area",
-                detail: "Average number of robbery offences per year near the postcode, from police.uk street-level crime data. Robbery involves theft with force or threat of force.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Vehicle crime (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly vehicle crime in the area",
-                detail: "Average number of vehicle crime incidents per year near the postcode, from police.uk street-level crime data. Includes theft of and from vehicles.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Anti-social behaviour (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly anti-social behaviour incidents in the area",
-                detail: "Average number of anti-social behaviour incidents per year near the postcode, from police.uk street-level crime data. Includes nuisance, environmental, and personal anti-social behaviour.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Criminal damage and arson (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly criminal damage and arson in the area",
-                detail: "Average number of criminal damage and arson incidents per year near the postcode, from police.uk street-level crime data.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Other theft (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly other theft offences in the area",
-                detail: "Average number of 'other theft' offences per year near the postcode, from police.uk street-level crime data. Includes theft not classified under burglary, vehicle crime, shoplifting, or bicycle theft.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Theft from the person (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly theft from the person in the area",
-                detail: "Average number of theft from the person offences per year near the postcode, from police.uk street-level crime data. Includes pickpocketing and bag snatching without force.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Shoplifting (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly shoplifting offences in the area",
-                detail: "Average number of shoplifting offences per year near the postcode, from police.uk street-level crime data.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Bicycle theft (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly bicycle theft in the area",
-                detail: "Average number of bicycle theft offences per year near the postcode, from police.uk street-level crime data.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Drugs (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly drug offences in the area",
-                detail: "Average number of drug offences per year near the postcode, from police.uk street-level crime data. Includes possession and trafficking offences.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Possession of weapons (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly weapons possession offences in the area",
-                detail: "Average number of possession of weapons offences per year near the postcode, from police.uk street-level crime data.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Public order (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly public order offences in the area",
-                detail: "Average number of public order offences per year near the postcode, from police.uk street-level crime data. Includes causing fear, alarm, or distress.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
-            Feature::Numeric(FeatureConfig {
-                name: "Other crime (avg/yr)",
-                bounds: Bounds::Percentile {
-                    low: 2.0,
-                    high: 98.0,
-                },
-                step: 1.0,
-                description: "Average yearly other crime in the area",
-                detail: "Average number of other crime offences per year near the postcode, from police.uk street-level crime data. A catch-all category for offences not classified elsewhere.",
-                source: "crime",
-                prefix: "",
-                suffix: "",
-                raw: false,
-                absolute: false,
-            }),
+        features: crime_features![
+            (
+                "Serious crime",
+                "Serious crime — violence, robbery, burglary and weapons possession — near the postcode"
+            ),
+            (
+                "Minor crime",
+                "Lower-severity crime — anti-social behaviour, theft, criminal damage, drugs and public order — near the postcode"
+            ),
+            (
+                "Violence and sexual offences",
+                "Violence and sexual offences (assault, harassment, sexual offences) near the postcode"
+            ),
+            ("Burglary", "Burglary (residential and commercial) near the postcode"),
+            ("Robbery", "Robbery (theft with force or threat) near the postcode"),
+            ("Vehicle crime", "Vehicle crime (theft of and from vehicles) near the postcode"),
+            ("Anti-social behaviour", "Anti-social behaviour near the postcode"),
+            ("Criminal damage and arson", "Criminal damage and arson near the postcode"),
+            (
+                "Other theft",
+                "Other theft (not burglary, vehicle, shoplifting or bicycle theft) near the postcode"
+            ),
+            (
+                "Theft from the person",
+                "Theft from the person (pickpocketing, bag snatching) near the postcode"
+            ),
+            ("Shoplifting", "Shoplifting near the postcode"),
+            ("Bicycle theft", "Bicycle theft near the postcode"),
+            ("Drugs", "Drug offences (possession and trafficking) near the postcode"),
+            ("Possession of weapons", "Possession of weapons near the postcode"),
+            (
+                "Public order",
+                "Public order offences (causing fear, alarm or distress) near the postcode"
+            ),
+            ("Other crime", "Other crime (offences not classified elsewhere) near the postcode"),
         ],
     },
     FeatureGroup {
@@ -826,9 +673,10 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
             // shares sum to ~100% per neighbourhood (LSOA) and render as a
             // stacked composition (see STACKED_GROUPS["Neighbours"] in the
             // frontend), like the ethnicity, qualifications and vote-share bars.
-            // Unlike those, the three shares are ALSO offered as individual
-            // filters (they are not added to the display-only skip-list in
-            // Filters.tsx), so users can target e.g. owner-occupier-heavy areas.
+            // Unlike those — each folded into a single dropdown filter that
+            // selects one band — the three tenure shares are offered as
+            // individual filters, so users can target e.g. owner-occupier-heavy
+            // areas.
             Feature::Numeric(FeatureConfig {
                 name: "% Owner occupied",
                 bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
@@ -1292,7 +1140,7 @@ mod tests {
             "Income Score",             // 0..100 percentile
             "% White",                  // 0..100 percentage
             "Noise (dB)",               // 50..80, range > threshold
-            "Serious crime (avg/yr)",   // Percentile bounds, fractional
+            "Serious crime (/yr, 7y)",  // Percentile bounds, fractional
             "Interior height (m)",      // step 0.1
             "Estimated current price",  // step 10000
         ] {

server-rs/src/main.rs

@@ -334,6 +334,18 @@ struct Cli {
     #[arg(long, env = "CRIME_BY_YEAR_PATH")]
     crime_by_year_path: PathBuf,
 
+    /// Path to the per-incident crime-records parquet (last 7 years, postcode-
+    /// sorted) backing the "individual crimes" list. Spilled to disk when
+    /// `--spill-dir` is set.
+    #[arg(long, env = "CRIME_RECORDS_PATH")]
+    crime_records_path: PathBuf,
+
+    /// Path to the precomputed national/per-outcode/per-sector crime-averages
+    /// parquet (built by pipeline.transform.area_crime_averages). The right pane
+    /// uses it to compare a selection's crime rates against its surroundings.
+    #[arg(long, env = "AREA_CRIME_AVERAGES_PATH")]
+    area_crime_averages_path: PathBuf,
+
     /// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
     /// residents; display-only side table for the right pane). Optional: when
     /// absent or missing, the area pane simply omits the population figure.
@@ -725,6 +737,16 @@ async fn main() -> anyhow::Result<()> {
         Arc::new(data)
     };
 
+    let crime_records = {
+        let path = &cli.crime_records_path;
+        if !path.exists() {
+            bail!("Crime-records parquet not found: {}", path.display());
+        }
+        let data = data::CrimeRecords::load(path, spill_dir)?;
+        trim_allocator("crime-records load");
+        Arc::new(data)
+    };
+
     let population = match &cli.population_path {
         Some(path) if path.exists() => {
             let data = data::PostcodePopulation::load(path)?;
@@ -742,13 +764,11 @@ async fn main() -> anyhow::Result<()> {
     };
 
     let area_crime_averages = {
-        let data = property_data.compute_area_crime_averages();
-        info!(
-            outcodes = data.by_outcode.len(),
-            sectors = data.by_sector.len(),
-            crime_types = data.crime_types.len(),
-            "Per-outcode/sector crime averages computed"
-        );
+        let path = &cli.area_crime_averages_path;
+        if !path.exists() {
+            bail!("Area-crime-averages parquet not found: {}", path.display());
+        }
+        let data = data::AreaCrimeAverages::load(path)?;
         trim_allocator("area crime averages");
         Arc::new(data)
     };
@@ -781,6 +801,7 @@ async fn main() -> anyhow::Result<()> {
         actual_listings,
         developments,
         crime_by_year,
+        crime_records,
         population,
         area_crime_averages,
         token_cache,
@@ -899,6 +920,10 @@ async fn main() -> anyhow::Result<()> {
             "/api/postcode-properties",
             get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
         )
+        .route(
+            "/api/crime-records",
+            get(routes::get_crime_records).layer(ConcurrencyLimitLayer::new(5)),
+        )
         .route(
             "/api/screenshot",
             get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),

server-rs/src/pocketbase.rs

@@ -1319,6 +1319,49 @@ pub async fn ensure_collections(
         ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?;
     }
 
+    // Per-user record of which property listings a user has opened, so visited listings
+    // can be drawn in a distinct colour on the map. One row per (user, url); the unique
+    // index makes re-clicks idempotent.
+    let clicked_listings_index =
+        "CREATE UNIQUE INDEX idx_clicked_listings_user_url ON clicked_listings (user, url)";
+    if !existing.iter().any(|n| n == "clicked_listings") {
+        let users_id = find_users_collection_id(client, base_url, &token).await?;
+        let user_only = Some("user = @request.auth.id".to_string());
+        create_collection(
+            client,
+            base_url,
+            &token,
+            CreateCollection {
+                name: "clicked_listings".to_string(),
+                r#type: "base".to_string(),
+                fields: vec![
+                    Field::relation("user", &users_id),
+                    Field::text("url", true),
+                    Field::autodate("created", true, false),
+                    Field::autodate("updated", true, true),
+                ],
+                list_rule: user_only.clone(),
+                view_rule: user_only.clone(),
+                create_rule: user_only.clone(),
+                update_rule: user_only.clone(),
+                delete_rule: user_only,
+                indexes: vec![clicked_listings_index.to_string()],
+            },
+        )
+        .await?;
+    } else {
+        ensure_user_owned_rules(client, base_url, &token, "clicked_listings").await?;
+        ensure_autodate_fields(client, base_url, &token, "clicked_listings").await?;
+        ensure_collection_indexes(
+            client,
+            base_url,
+            &token,
+            "clicked_listings",
+            &[("idx_clicked_listings_user_url", clicked_listings_index)],
+        )
+        .await?;
+    }
+
     Ok(())
 }
 

server-rs/src/routes.rs

@@ -1,6 +1,7 @@
 mod actual_listings;
 mod ai_filters;
 mod checkout;
+mod crime_records;
 mod developments;
 mod export;
 mod features;
@@ -35,6 +36,7 @@ pub(crate) mod travel_time;
 pub use actual_listings::get_actual_listings;
 pub use ai_filters::{build_system_prompt, post_ai_filters};
 pub use checkout::post_checkout;
+pub use crime_records::get_crime_records;
 pub use developments::get_developments;
 pub use export::get_export;
 pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};

server-rs/src/routes/ai_filters/parsing.rs

@@ -336,8 +336,8 @@ mod tests {
             "Good+ primary school catchments"
         );
         assert_eq!(
-            canonical_filter_name("Specific crimes:Burglary%20%28avg%2Fyr%29:1"),
-            "Burglary (avg/yr)"
+            canonical_filter_name("Specific crimes:Burglary%20%28%2Fyr%2C%207y%29:1"),
+            "Burglary (/yr, 7y)"
         );
         assert_eq!(
             canonical_filter_name("Political vote share:%25%20Labour:0"),

server-rs/src/routes/ai_filters/prompt.rs

@@ -25,9 +25,9 @@ pub fn build_system_prompt(
            or \"max\" (at most this value). Never set two filters on the same feature.\n\
          - Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
          - \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
-         - \"low crime\" / \"safe\" = low values on the Serious crime (avg/yr) and Minor crime (avg/yr) \
-           features (area-normalised incident density near the postcode). Prefer these aggregates for broad \
-           area safety; use specific crime features only when the user names a crime type.\n\
+         - \"low crime\" / \"safe\" = low values on the Serious crime (/yr, 7y) and Minor crime (/yr, 7y) \
+           features (average recorded incidents per year near the postcode, last 7 years). Prefer these aggregates for broad \
+           area safety; use specific crime features only when the user names a crime type. Use a \"(/yr, 2y)\" feature only when the user asks about recent crime.\n\
          - \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \
            or low Distance to nearest park (km), depending on wording.\n\
          - \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
@@ -171,8 +171,8 @@ pub fn build_system_prompt(
     parts.push(
         "\nUser: \"safe quiet area with good schools and parks\"\n\
          Output: {\"numeric_filters\": [\
-         {\"name\": \"Serious crime (avg/yr)\", \"bound\": \"max\", \"value\": 5}, \
-         {\"name\": \"Minor crime (avg/yr)\", \"bound\": \"max\", \"value\": 20}, \
+         {\"name\": \"Serious crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 5}, \
+         {\"name\": \"Minor crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 20}, \
          {\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
          {\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \
          {\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \
@@ -237,7 +237,7 @@ pub fn build_system_prompt(
         "\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\
          Output: {\"numeric_filters\": [\
          {\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \
-         {\"name\": \"Burglary (avg/yr)\", \"bound\": \"max\", \"value\": 10}, \
+         {\"name\": \"Burglary (/yr, 7y)\", \"bound\": \"max\", \"value\": 10}, \
          {\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \
          \"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
             .to_string(),

server-rs/src/routes/crime_records.rs

@@ -0,0 +1,194 @@
+//! `GET /api/crime-records` — the individual police.uk crimes (last 7 years)
+//! behind a selected hexagon or postcode, paginated. Display-only and
+//! independent of the property filters, like the population figure: the records
+//! are an attribute of the area, not of the filter-matching subset.
+
+use std::str::FromStr;
+use std::sync::Arc;
+
+use axum::extract::{Query, State};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Json};
+use axum::Extension;
+use rustc_hash::{FxHashMap, FxHashSet};
+use serde::{Deserialize, Serialize};
+use tracing::{info, warn};
+
+use crate::auth::OptionalUser;
+use crate::licensing::{check_license_bounds, check_license_point, resolve_share_code};
+use crate::parsing::{cell_for_row_cached, h3_cell_bounds, needs_parent, validate_h3_resolution};
+use crate::state::SharedState;
+use crate::utils::normalize_postcode;
+
+/// Default and hard-cap page sizes for the records list.
+const DEFAULT_LIMIT: usize = 200;
+const MAX_LIMIT: usize = 500;
+
+#[derive(Deserialize)]
+pub struct CrimeRecordsParams {
+    /// Hexagon selection: H3 cell + resolution. Mutually exclusive with `postcode`.
+    pub h3: Option<String>,
+    pub resolution: Option<u8>,
+    /// Postcode selection.
+    pub postcode: Option<String>,
+    pub offset: Option<usize>,
+    pub limit: Option<usize>,
+    /// Lower bound on `month_index` (`year*12 + month0`) to restrict to a recent
+    /// window; omitted = all stored records (last 7 years).
+    pub since: Option<u32>,
+    /// Share-link code; grants scoped access for unlicensed users.
+    pub share: Option<String>,
+}
+
+#[derive(Serialize)]
+pub struct CrimeRecord {
+    /// `"YYYY-MM"`.
+    pub month: String,
+    #[serde(rename = "type")]
+    pub crime_type: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub location: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub outcome: Option<String>,
+    pub lat: f32,
+    pub lon: f32,
+}
+
+#[derive(Serialize)]
+pub struct CrimeRecordsResponse {
+    pub records: Vec<CrimeRecord>,
+    pub total: usize,
+    pub offset: usize,
+    pub truncated: bool,
+}
+
+fn format_month(month_index: u32) -> String {
+    let year = month_index / 12;
+    let month = month_index % 12 + 1;
+    format!("{year:04}-{month:02}")
+}
+
+pub async fn get_crime_records(
+    State(shared): State<Arc<SharedState>>,
+    Extension(user): Extension<OptionalUser>,
+    Extension(geo): Extension<crate::demo_zone::DemoZone>,
+    Query(params): Query<CrimeRecordsParams>,
+) -> Result<Json<CrimeRecordsResponse>, axum::response::Response> {
+    let state = shared.load_state();
+    let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
+    let offset = params.offset.unwrap_or(0);
+    let limit = params.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
+    let since = params.since;
+
+    // Resolve the selection to a set of postcodes, after a license check scoped
+    // to the selection's geometry (bounds for a hexagon, point for a postcode).
+    enum Selection {
+        Hexagon { cell: u64, resolution: u8 },
+        Postcode(String),
+    }
+
+    let selection = if let Some(h3) = params.h3.clone() {
+        let cell = h3o::CellIndex::from_str(&h3).map_err(|error| {
+            warn!(h3 = %h3, error = %error, "Invalid H3 cell index");
+            (StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {error}")).into_response()
+        })?;
+        let resolution = params.resolution.ok_or_else(|| {
+            (StatusCode::BAD_REQUEST, "resolution is required with h3").into_response()
+        })?;
+        validate_h3_resolution(resolution).map_err(IntoResponse::into_response)?;
+        let bounds = h3_cell_bounds(cell, 0.0);
+        check_license_bounds(&user.0, bounds, geo.free_zone, share_bounds)?;
+        Selection::Hexagon {
+            cell: cell.into(),
+            resolution,
+        }
+    } else if let Some(postcode) = params.postcode.clone() {
+        let normalized = normalize_postcode(&postcode);
+        let &pc_idx = state
+            .postcode_data
+            .postcode_to_idx
+            .get(&normalized)
+            .ok_or_else(|| {
+                (StatusCode::NOT_FOUND, format!("Postcode not found: {normalized}")).into_response()
+            })?;
+        let (lat, lon) = state.postcode_data.centroids[pc_idx];
+        check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
+        Selection::Postcode(normalized)
+    } else {
+        return Err((StatusCode::BAD_REQUEST, "h3 or postcode is required").into_response());
+    };
+
+    let result = tokio::task::spawn_blocking(move || -> Result<CrimeRecordsResponse, String> {
+        // Distinct postcodes covered by the selection.
+        let postcodes: Vec<String> = match selection {
+            Selection::Postcode(pc) => vec![pc],
+            Selection::Hexagon { cell, resolution } => {
+                let h3_res = h3o::Resolution::try_from(resolution)
+                    .map_err(|err| format!("Invalid H3 resolution {resolution}: {err}"))?;
+                let need_parent = needs_parent(resolution);
+                let h3o_cell = h3o::CellIndex::try_from(cell)
+                    .map_err(|err| format!("Invalid H3 cell: {err}"))?;
+                let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(h3o_cell, 0.001);
+                let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
+                let mut seen: FxHashSet<&str> = FxHashSet::default();
+                let mut out: Vec<String> = Vec::new();
+                state.grid.for_each_in_bounds(
+                    min_lat,
+                    min_lon,
+                    max_lat,
+                    max_lon,
+                    |row_idx| {
+                        let row = row_idx as usize;
+                        if cell_for_row_cached(
+                            row,
+                            &state.h3_cells,
+                            h3_res,
+                            need_parent,
+                            &mut h3_cache,
+                        ) == cell
+                        {
+                            let pc = state.data.postcode(row);
+                            if seen.insert(pc) {
+                                out.push(pc.to_string());
+                            }
+                        }
+                    },
+                );
+                out
+            }
+        };
+
+        let pc_refs: Vec<&str> = postcodes.iter().map(String::as_str).collect();
+        let indices = state.crime_records.gather(&pc_refs, since);
+        let total = indices.len();
+        let records: Vec<CrimeRecord> = indices
+            .iter()
+            .skip(offset)
+            .take(limit)
+            .map(|&idx| {
+                let v = state.crime_records.view(idx);
+                CrimeRecord {
+                    month: format_month(v.month_index),
+                    crime_type: v.crime_type.to_string(),
+                    location: v.location.map(str::to_string),
+                    outcome: v.outcome.map(str::to_string),
+                    lat: v.lat,
+                    lon: v.lon,
+                }
+            })
+            .collect();
+        let truncated = offset + records.len() < total;
+        Ok(CrimeRecordsResponse {
+            records,
+            total,
+            offset,
+            truncated,
+        })
+    })
+    .await
+    .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response())?
+    .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error).into_response())?;
+
+    info!(total = result.total, returned = result.records.len(), "GET /api/crime-records");
+    Ok(Json(result))
+}

server-rs/src/routes/hexagon_stats.rs

@@ -74,20 +74,20 @@ pub struct CrimeYearPoint {
 
 #[derive(Serialize)]
 pub struct CrimeYearStats {
-    /// Underlying crime type (e.g. "Burglary"). Matches existing crime feature
-    /// names with the `" (avg/yr)"` suffix stripped.
+    /// Underlying crime type, bare (e.g. "Burglary"). Matches the type prefix of
+    /// the `"(/yr, …)"` crime features.
     pub name: String,
     pub points: Vec<CrimeYearPoint>,
 }
 
-/// Average headline crime rate (avg/yr) for one crime type across the
-/// selection's outcode and postcode sector. Comparable to the national average
-/// shown per metric in the right pane.
+/// Average crime count for one crime feature across the selection's outcode and
+/// postcode sector. Comparable to the national average shown per metric in the
+/// right pane.
 #[derive(Serialize)]
 pub struct CrimeAreaAverage {
-    /// Crime type, bare (e.g. "Burglary"). Matches `CrimeYearStats.name`.
+    /// Full crime-feature name (e.g. "Burglary (/yr, 7y)").
     pub name: String,
-    /// Exact national mean (avg/yr) — the frontend prefers this over the
+    /// Exact national mean count — the frontend prefers this over the
     /// histogram-bin national average for crime so all four numbers in the row
     /// share one estimator.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -161,10 +161,15 @@ pub struct HexagonStatsResponse {
     /// present only when sector crime averages are available for it.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub crime_sector: Option<String>,
-    /// Per-crime-type average rates across the central postcode's outcode and
+    /// Per-crime-type average counts across the central postcode's outcode and
     /// sector, shown alongside the national average for each crime metric.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub crime_area_averages: Vec<CrimeAreaAverage>,
+    /// Total individual crime records (last 7 years) across the distinct
+    /// postcodes in this selection — the count behind the "individual crimes"
+    /// list. Filter-independent, like `population`.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub crime_total_records: Option<u32>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub central_postcode: Option<String>,
     /// Total usual residents (ONS Census 2021) living across the distinct
@@ -699,26 +704,42 @@ pub async fn get_hexagon_stats(
             &field_set,
         );
 
-        // Sum usual residents across the distinct postcodes covered by the
-        // hexagon. Computed over `area_rows` (all properties in the cell), not
-        // the filter-matching subset, so toggling filters never changes it —
-        // population is an attribute of the area, like the council-house count.
+        // Distinct postcodes covered by the hexagon, taken over `area_rows` (all
+        // properties in the cell), not the filter-matching subset — population and
+        // the crime-records count are attributes of the area, independent of the
+        // active filters (like the council-house count).
+        let mut seen: HashSet<&str> = HashSet::new();
+        let mut area_postcodes: Vec<&str> = Vec::new();
+        for &row in &area_rows {
+            let pc = state.data.postcode(row);
+            if seen.insert(pc) {
+                area_postcodes.push(pc);
+            }
+        }
+
         let population = {
-            let mut seen: HashSet<&str> = HashSet::new();
             let mut total: u64 = 0;
             let mut found = false;
-            for &row in &area_rows {
-                let pc = state.data.postcode(row);
-                if seen.insert(pc) {
-                    if let Some(p) = state.population.for_postcode(pc) {
-                        total += p as u64;
-                        found = true;
-                    }
+            for &pc in &area_postcodes {
+                if let Some(p) = state.population.for_postcode(pc) {
+                    total += p as u64;
+                    found = true;
                 }
             }
             found.then(|| total.min(u32::MAX as u64) as u32)
         };
 
+        let crime_total_records = {
+            // Sum the per-postcode counts straight from the CSR index instead of
+            // materializing (and sorting) every record index: this keeps the
+            // mmap-backed columns cold on the hot hexagon path.
+            let total: u64 = area_postcodes
+                .iter()
+                .map(|pc| state.crime_records.total_for(pc) as u64)
+                .sum();
+            (total > 0).then(|| total.min(u32::MAX as u64) as u32)
+        };
+
         Ok(HexagonStatsResponse {
             count: total_count,
             numeric_features,
@@ -729,6 +750,7 @@ pub async fn get_hexagon_stats(
             crime_outcode,
             crime_sector,
             crime_area_averages,
+            crime_total_records,
             central_postcode,
             population,
             filter_exclusions,

server-rs/src/routes/postcode_stats.rs

@@ -227,6 +227,11 @@ pub async fn get_postcode_stats(
         // Usual residents (Census 2021) for this postcode. Display-only.
         let population = state.population.for_postcode(&postcode_str);
 
+        let crime_total_records = {
+            let total = state.crime_records.total_for(&postcode_str);
+            (total > 0).then_some(total)
+        };
+
         Ok(HexagonStatsResponse {
             count: total_count,
             numeric_features,
@@ -237,6 +242,7 @@ pub async fn get_postcode_stats(
             crime_outcode,
             crime_sector,
             crime_area_averages,
+            crime_total_records,
             central_postcode: None,
             population,
             filter_exclusions,

server-rs/src/routes/shorten.rs

@@ -127,15 +127,20 @@ fn is_allowed_param_key(key: &str) -> bool {
             | "filter"
             | "school"
             | "crime"
+            | "crimeSeverity"
             | "voteShare"
             | "ethnicity"
+            | "qualification"
+            | "tenure"
             | "amenityDistance"
             | "transportDistance"
             | "amenityCount2km"
             | "amenityCount5km"
             | "poi"
             | "overlay"
+            | "crimeType"
             | "basemap"
+            | "colorOpacity"
             | "tab"
             | "pc"
             | "tt"
@@ -594,6 +599,22 @@ mod tests {
         assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
     }
 
+    #[test]
+    fn preserves_all_filter_params_for_share_links() {
+        // Every filter param emitted by the frontend's stateToParams() must survive
+        // shortening; an unsupported key is rejected outright (see is_allowed_param_key),
+        // which fails the whole share link rather than dropping a single filter.
+        // Values use %3A (":") since form re-serialization keeps it stable.
+        let query = "crimeSeverity=Serious%3A0%3A5\
+            &qualification=Degree%3A20%3A80\
+            &tenure=Owner%3A30%3A90\
+            &crimeType=burglary\
+            &colorOpacity=60";
+        let params = sanitized_query_params(query, false).unwrap();
+
+        assert_eq!(params, query);
+    }
+
     #[test]
     fn escapes_html_attributes() {
         assert_eq!(escape_attr(r#""'><&"#), "&quot;&#39;&gt;&lt;&amp;");

server-rs/src/routes/stats.rs

@@ -11,8 +11,8 @@ use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
 use crate::utils::{postcode_outcode, postcode_sector};
 
 use super::hexagon_stats::{
-    CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats,
-    NumericFeatureStats, PricePoint,
+    CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats,
+    HistogramStats, NumericFeatureStats, PricePoint,
 };
 
 /// Extract price history (year, price) pairs from matching rows, downsampled if needed.
@@ -352,11 +352,14 @@ pub fn compute_crime_by_year(
     let mut out = Vec::new();
     for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() {
         // Crime types in the by-year side table are bare (e.g. "Burglary"), while
-        // the configured feature names carry an " (avg/yr)" suffix. Match either
-        // form so callers can pass the feature names they already know.
+        // the configured feature names carry a window suffix ("Burglary (/yr,
+        // 7y)"). Emit the bare-type trend if the bare name is requested directly or
+        // any of its windowed features is.
         if fields_specified {
-            let with_suffix = format!("{name} (avg/yr)");
-            if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
+            let prefix = format!("{name} (");
+            if !field_set.contains(name.as_str())
+                && !field_set.iter().any(|f| f.starts_with(&prefix))
+            {
                 continue;
             }
         }
@@ -395,6 +398,7 @@ pub fn compute_crime_by_year(
     out
 }
 
+
 /// Latest year present anywhere in the by-year crime dataset. The client
 /// compares each selection's last charted year against this to caption
 /// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as
@@ -440,13 +444,10 @@ pub fn area_crime_averages_for(
 
     let mut out = Vec::new();
     for (idx, name) in averages.crime_types.iter().enumerate() {
-        // Crime types are bare here ("Burglary"); requested fields may carry the
-        // " (avg/yr)" suffix — accept either form, like compute_crime_by_year.
-        if fields_specified {
-            let with_suffix = format!("{name} (avg/yr)");
-            if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
-                continue;
-            }
+        // `name` is the full crime-feature name here (e.g. "Burglary (/yr,
+        // 7y)"), matching exactly the feature fields the caller requests.
+        if fields_specified && !field_set.contains(name.as_str()) {
+            continue;
         }
         let national_val = finite_at(Some(&averages.national), idx);
         let outcode_val = finite_at(outcode_means, idx);
@@ -595,7 +596,10 @@ mod tests {
         let mut by_sector = rustc_hash::FxHashMap::default();
         by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
         AreaCrimeAverages {
-            crime_types: vec!["Burglary".to_string(), "Robbery".to_string()],
+            crime_types: vec![
+                "Burglary (/yr, 7y)".to_string(),
+                "Robbery (/yr, 7y)".to_string(),
+            ],
             national: vec![8.0, 6.0],
             by_outcode,
             by_sector,
@@ -611,12 +615,18 @@ mod tests {
         assert_eq!(sector.as_deref(), Some("E14 2"));
         assert_eq!(out.len(), 2);
 
-        let burglary = out.iter().find(|c| c.name == "Burglary").unwrap();
+        let burglary = out
+            .iter()
+            .find(|c| c.name == "Burglary (/yr, 7y)")
+            .unwrap();
         assert_eq!(burglary.national, Some(8.0));
         assert_eq!(burglary.outcode, Some(10.0));
         assert_eq!(burglary.sector, Some(5.0));
 
-        let robbery = out.iter().find(|c| c.name == "Robbery").unwrap();
+        let robbery = out
+            .iter()
+            .find(|c| c.name == "Robbery (/yr, 7y)")
+            .unwrap();
         assert_eq!(robbery.national, Some(6.0));
         // The outcode value was NaN — dropped to None; the sector value is finite.
         assert_eq!(robbery.outcode, None);
@@ -626,11 +636,13 @@ mod tests {
     #[test]
     fn area_crime_averages_respect_fields_filter() {
         let avgs = sample_averages();
-        // The suffixed feature-name form is accepted, like compute_crime_by_year.
-        let fields: HashSet<String> = ["Burglary (avg/yr)".to_string()].into_iter().collect();
+        // Area averages are keyed by the full crime-feature name.
+        let fields: HashSet<String> = ["Burglary (/yr, 7y)".to_string()]
+            .into_iter()
+            .collect();
         let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
         assert_eq!(out.len(), 1);
-        assert_eq!(out[0].name, "Burglary");
+        assert_eq!(out[0].name, "Burglary (/yr, 7y)");
     }
 
     #[test]

server-rs/src/state.rs

@@ -6,9 +6,9 @@ use rustc_hash::FxHashMap;
 use crate::auth::TokenCache;
 use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
 use crate::data::{
-    ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData,
-    POICategoryGroup, POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData,
-    TravelTimeStore,
+    ActualListingData, AreaCrimeAverages, CrimeByYearData, CrimeRecords,
+    DevelopmentData, OutcodeData, POICategoryGroup, POIData, PlaceData, PostcodeData,
+    PostcodePopulation, PropertyData, TravelTimeStore,
 };
 use crate::licensing::ShareBoundsCache;
 use crate::pocketbase::SuperuserTokenCache;
@@ -52,10 +52,13 @@ pub struct AppState {
     pub developments: Arc<DevelopmentData>,
     /// Per-LSOA per-year crime counts used by the right pane to plot trends.
     pub crime_by_year: Arc<CrimeByYearData>,
+    /// Per-postcode individual crime records (last 7 years), spill-backed,
+    /// served by the `/api/crime-records` endpoint and counted in stats.
+    pub crime_records: Arc<CrimeRecords>,
     /// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
     /// right pane. Display-only — never filterable. Empty when no data is loaded.
     pub population: Arc<PostcodePopulation>,
-    /// Precomputed per-outcode and per-postcode-sector average crime rates,
+    /// Precomputed per-outcode and per-postcode-sector average crime counts,
     /// shown in the right pane alongside the national average for each metric.
     pub area_crime_averages: Arc<AreaCrimeAverages>,
     /// Token validation cache (60s TTL)
@@ -178,6 +181,7 @@ impl AppState {
                 series_by_postcode: FxHashMap::default(),
                 covered_years_by_postcode: FxHashMap::default(),
             }),
+            crime_records: Arc::new(CrimeRecords::empty()),
             population: Arc::new(PostcodePopulation::empty()),
             area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
             token_cache: Arc::new(TokenCache::new()),