This commit is contained in:
Andras Schmelczer 2026-06-25 22:29:52 +01:00
parent 2efa4d9f47
commit 5e73287eaf
99 changed files with 6392 additions and 1462 deletions

View file

@ -56,4 +56,4 @@ EXPOSE 8001
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \ HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
CMD curl -f http://localhost:8001/health || exit 1 CMD curl -f http://localhost:8001/health || exit 1
ENTRYPOINT ["./property-map-server"] ENTRYPOINT ["./property-map-server"]
CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--satellite-tiles", "/app/data/satellite.pmtiles", "--satellite-highres-tiles", "/app/data/satellite_highres.pmtiles", "--noise-overlay-tiles", "/app/data/noise_lden_10m.pmtiles", "--crime-hotspot-tiles", "/app/data/crime_hotspots.pmtiles", "--tree-overlay-tiles", "/app/data/trees_outside_woodlands.pmtiles", "--property-border-tiles", "/app/data/property_borders.pmtiles", "--crime-by-year-path", "/app/data/crime_by_postcode_by_year.parquet", "--area-crime-averages-path", "/app/data/area_crime_averages.parquet", "--population-path", "/app/data/population_by_postcode.parquet", "--developments-path", "/app/data/development_sites.parquet", "--dist", "/app/frontend/dist"] CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--satellite-tiles", "/app/data/satellite.pmtiles", "--satellite-highres-tiles", "/app/data/satellite_highres.pmtiles", "--noise-overlay-tiles", "/app/data/noise_lden_10m.pmtiles", "--crime-hotspot-tiles", "/app/data/crime_hotspots.pmtiles", "--tree-overlay-tiles", "/app/data/trees_outside_woodlands.pmtiles", "--property-border-tiles", "/app/data/property_borders.pmtiles", "--crime-by-year-path", "/app/data/crime_by_postcode_by_year.parquet", "--crime-records-path", "/app/data/crime_records.parquet", "--area-crime-averages-path", "/app/data/area_crime_averages.parquet", "--population-path", "/app/data/population_by_postcode.parquet", "--developments-path", "/app/data/development_sites.parquet", "--dist", "/app/frontend/dist"]

View file

@ -27,7 +27,10 @@ POSTCODES_RAW := $(DATA_DIR)/gb-postcodes-v5
POSTCODES_PQ := $(DATA_DIR)/postcode.parquet POSTCODES_PQ := $(DATA_DIR)/postcode.parquet
PROPERTIES_PQ := $(DATA_DIR)/properties.parquet PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
MERGE_STAMP := $(DATA_DIR)/.merge_done MERGE_STAMP := $(DATA_DIR)/.merge_done
PRICE_INPUTS := $(DATA_DIR)/price_inputs.parquet
POSTCODE_CENTROIDS := $(DATA_DIR)/postcode_centroids.parquet
PRICE_INDEX := $(DATA_DIR)/price_index.parquet PRICE_INDEX := $(DATA_DIR)/price_index.parquet
PRICE_ESTIMATES := $(DATA_DIR)/price_estimates.parquet
PRICES_STAMP := $(DATA_DIR)/.prices_done PRICES_STAMP := $(DATA_DIR)/.prices_done
EPC := $(MANUAL_DATA)/domestic-csv.zip EPC := $(MANUAL_DATA)/domestic-csv.zip
ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet
@ -38,6 +41,8 @@ TENURE := $(DATA_DIR)/tenure_by_lsoa.parquet
CRIME_DIR := $(DATA_DIR)/crime CRIME_DIR := $(DATA_DIR)/crime
CRIME := $(DATA_DIR)/crime_by_postcode.parquet CRIME := $(DATA_DIR)/crime_by_postcode.parquet
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet
CRIME_RECORDS := $(DATA_DIR)/crime_records.parquet
AREA_CRIME_AVERAGES := $(DATA_DIR)/area_crime_averages.parquet
POPULATION := $(DATA_DIR)/population_by_postcode.parquet POPULATION := $(DATA_DIR)/population_by_postcode.parquet
CRIME_STAMP := $(CRIME_DIR)/.downloaded CRIME_STAMP := $(CRIME_DIR)/.downloaded
NOISE := $(DATA_DIR)/road_noise.parquet NOISE := $(DATA_DIR)/road_noise.parquet
@ -91,8 +96,11 @@ VALIDATE_OUTPUTS := uv run python -m pipeline.validate_outputs
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
MERGE_DEPS := pipeline/transform/merge.py MERGE_DEPS := pipeline/transform/merge.py
AREA_CRIME_AVERAGES_DEPS := pipeline/transform/area_crime_averages.py
PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py
PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py
PRICE_JOIN_DEPS := pipeline/transform/join_price_estimates.py pipeline/transform/price_estimation/utils.py
PRICE_INPUTS_DEPS := pipeline/transform/property_base.py pipeline/utils/postcode_mapping.py
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \ PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \
pipeline/transform/postcode_boundaries/greenspace.py \ pipeline/transform/postcode_boundaries/greenspace.py \
@ -116,11 +124,11 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
download-postcodes download-noise download-inspire download-crime \ download-postcodes download-noise download-inspire download-crime \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \ download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \
download-map-assets \ download-map-assets \
transform-pois transform-epc-pp transform-crime transform-poi-proximity \ transform-pois transform-epc-pp transform-crime transform-poi-proximity transform-area-crime-averages \
transform-school-catchments transform-tree-density \ transform-school-catchments transform-tree-density \
generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX) prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) $(CRIME_RECORDS) $(AREA_CRIME_AVERAGES) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX) $(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ) merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" $(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
@ -178,6 +186,7 @@ transform-pois: $(POIS_FILTERED)
transform-epc-pp: $(EPC_PP) transform-epc-pp: $(EPC_PP)
transform-crime: $(CRIME) transform-crime: $(CRIME)
transform-poi-proximity: $(POI_PROXIMITY) transform-poi-proximity: $(POI_PROXIMITY)
transform-area-crime-averages: $(AREA_CRIME_AVERAGES)
transform-school-catchments: $(SCHOOL_CATCH) transform-school-catchments: $(SCHOOL_CATCH)
transform-tree-density: $(TREE_DENSITY_PC) transform-tree-density: $(TREE_DENSITY_PC)
generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP) generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP)
@ -383,9 +392,9 @@ $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(GIAS) $(OFSTE
$(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py $(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@ uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py $(CRIME) $(CRIME_BY_YEAR) $(CRIME_RECORDS) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv" $(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR) uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR) --output-records $(CRIME_RECORDS)
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS) $(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@ uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
@ -450,16 +459,42 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
$(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP) $(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" $(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
$(PRICE_INDEX): $(MERGE_STAMP) $(PRICE_INDEX_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ) # Slim price-estimation inputs, built straight from epc_pp + arcgis (NOT the
uv run python -m pipeline.transform.price_estimation.index --input $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --output $@ # merge outputs). This is what decouples the expensive index/kNN from merge:
# adding an area or property column to merge does not change these files, so the
# price index and estimates are reused and only the cheap join re-runs.
$(PRICE_INPUTS) $(POSTCODE_CENTROIDS) &: $(EPC_PP) $(ARCGIS) $(PRICE_INPUTS_DEPS)
uv run python -m pipeline.transform.property_base --epc-pp $(EPC_PP) --arcgis $(ARCGIS) --output $(PRICE_INPUTS) --centroids $(POSTCODE_CENTROIDS)
$(VALIDATE_OUTPUTS) --parquet $(PRICE_INPUTS) --parquet $(POSTCODE_CENTROIDS)
$(PRICE_INDEX): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX_DEPS)
uv run python -m pipeline.transform.price_estimation.index --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --output $@
$(VALIDATE_OUTPUTS) --parquet $@ $(VALIDATE_OUTPUTS) --parquet $@
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ) # Estimate prices on the slim inputs and write a standalone price_estimates.parquet
# (natural key + the two estimate columns). Never touches properties.parquet.
$(PRICE_ESTIMATES): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS)
uv run python -m pipeline.transform.price_estimation.estimate --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --index $(PRICE_INDEX) --output $@
$(VALIDATE_OUTPUTS) --parquet $@
# Join the estimate columns back onto properties.parquet by natural key
# (idempotent, atomic). Re-runs whenever merge or the estimates change.
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_ESTIMATES) $(PRICE_JOIN_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ) $(PRICE_INDEX)
@rm -f $@ @rm -f $@
uv run python -m pipeline.transform.price_estimation.estimate --properties $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --index $(PRICE_INDEX) uv run python -m pipeline.transform.join_price_estimates --properties $(PROPERTIES_PQ) --estimates $(PRICE_ESTIMATES)
$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX) $(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
@touch $@ @touch $@
# ── Area crime averages (post-merge) ────────────────────
# National / per-outcode / per-sector property-weighted mean headline crime
# rates for the right pane. Depends on $(PRICES_STAMP) so it reads the
# finalised properties.parquet (the price-estimate join rewrites it); only the
# per-postcode property counts and the crime values from postcode.parquet are
# used, both unaffected by that join.
$(AREA_CRIME_AVERAGES): $(PRICES_STAMP) $(AREA_CRIME_AVERAGES_DEPS) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
uv run python -m pipeline.transform.area_crime_averages --postcodes $(POSTCODES_PQ) --properties $(PROPERTIES_PQ) --output $@
$(VALIDATE_OUTPUTS) --parquet $@
$(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \ $(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
$(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \ $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \ $(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \

View file

@ -11,7 +11,7 @@ services:
command: > command: >
bash -c " bash -c "
cargo install cargo-watch && cargo install cargo-watch &&
cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data/properties.parquet --postcode-features /app/property-data/postcode.parquet --pois /app/property-data/filtered_uk_pois.parquet --places /app/property-data/places.parquet --tiles /app/property-data/uk.pmtiles --postcodes /app/property-data/postcode_boundaries --travel-times /app/property-data/travel-times --satellite-tiles /app/property-data/satellite.pmtiles --satellite-highres-tiles /app/property-data/satellite_highres.pmtiles --noise-overlay-tiles /app/property-data/noise_lden_10m.pmtiles --crime-hotspot-tiles /app/property-data/crime_hotspots.pmtiles --tree-overlay-tiles /app/property-data/trees_outside_woodlands.pmtiles --property-border-tiles /app/property-data/property_borders.pmtiles --crime-by-year-path /app/property-data/crime_by_postcode_by_year.parquet --population-path /app/property-data/population_by_postcode.parquet' cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data/properties.parquet --postcode-features /app/property-data/postcode.parquet --pois /app/property-data/filtered_uk_pois.parquet --places /app/property-data/places.parquet --tiles /app/property-data/uk.pmtiles --postcodes /app/property-data/postcode_boundaries --travel-times /app/property-data/travel-times --satellite-tiles /app/property-data/satellite.pmtiles --satellite-highres-tiles /app/property-data/satellite_highres.pmtiles --noise-overlay-tiles /app/property-data/noise_lden_10m.pmtiles --crime-hotspot-tiles /app/property-data/crime_hotspots.pmtiles --tree-overlay-tiles /app/property-data/trees_outside_woodlands.pmtiles --property-border-tiles /app/property-data/property_borders.pmtiles --crime-by-year-path /app/property-data/crime_by_postcode_by_year.parquet --crime-records-path /app/property-data/crime_records.parquet --area-crime-averages-path /app/property-data/area_crime_averages.parquet --population-path /app/property-data/population_by_postcode.parquet'
" "
ports: ports:
- "8001:8001" - "8001:8001"

View file

@ -383,7 +383,7 @@ export function SavedPage({
}) })
.catch((err) => { .catch((err) => {
if (!cancelled) { if (!cancelled) {
setShareLinksError(err instanceof Error ? err.message : 'Failed to fetch share links'); setShareLinksError(err instanceof Error ? err.message : t('accountPage.fetchShareLinksError'));
} }
}) })
.finally(() => { .finally(() => {
@ -393,7 +393,7 @@ export function SavedPage({
return () => { return () => {
cancelled = true; cancelled = true;
}; };
}, []); }, [t]);
const tabClass = (tab: string) => const tabClass = (tab: string) =>
`px-4 py-2 text-sm font-medium border-b-2 transition-colors ${ `px-4 py-2 text-sm font-medium border-b-2 transition-colors ${
@ -738,7 +738,7 @@ function InviteSection({ user }: { user: AuthUser }) {
setInviteUrl((prev) => ({ ...prev, [type]: data.url })); setInviteUrl((prev) => ({ ...prev, [type]: data.url }));
fetchInviteHistory(); fetchInviteHistory();
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to create invite'; const msg = err instanceof Error ? err.message : t('invitesPage.createInviteError');
setInviteError((prev) => ({ ...prev, [type]: msg })); setInviteError((prev) => ({ ...prev, [type]: msg }));
} finally { } finally {
setCreatingInvite((prev) => ({ ...prev, [type]: false })); setCreatingInvite((prev) => ({ ...prev, [type]: false }));
@ -931,7 +931,7 @@ export default function AccountPage({
assertOk(res, 'Update newsletter'); assertOk(res, 'Update newsletter');
await onRefreshAuth(); await onRefreshAuth();
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to update newsletter'; const msg = err instanceof Error ? err.message : t('accountPage.updateNewsletterError');
setNewsletterError(msg); setNewsletterError(msg);
} finally { } finally {
setNewsletterSaving(false); setNewsletterSaving(false);

View file

@ -150,7 +150,7 @@ function ProductDemoVideo() {
<track <track
kind="captions" kind="captions"
srcLang={(i18n.language ?? 'en').split('-')[0]} srcLang={(i18n.language ?? 'en').split('-')[0]}
label="Captions" label={t('common.captions')}
src={`/video/${productDemoSlug}.vtt`} src={`/video/${productDemoSlug}.vtt`}
/> />
</video> </video>

View file

@ -74,7 +74,7 @@ const DEMO_FEATURES: FeatureMeta[] = [
prefix: '£', prefix: '£',
}, },
{ {
name: 'Serious crime (avg/yr)', name: 'Serious crime (/yr, 7y)',
type: 'numeric', type: 'numeric',
group: 'Crime', group: 'Crime',
min: 0, min: 0,
@ -763,7 +763,7 @@ function RightPaneOnlyScreen({
<span className="truncate">{t('home.showcasePoliticalVoteShare')}</span> <span className="truncate">{t('home.showcasePoliticalVoteShare')}</span>
</div> </div>
<span className="shrink-0 text-xs font-black text-teal-700 dark:text-teal-300"> <span className="shrink-0 text-xs font-black text-teal-700 dark:text-teal-300">
2024 GE {t('home.showcaseGe2024')}
</span> </span>
</div> </div>
<StackedBarChart <StackedBarChart

View file

@ -155,11 +155,11 @@ export default function InvitePage({
window.location.href = data.checkout_url; window.location.href = data.checkout_url;
} }
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : 'Failed to redeem invite'); setError(err instanceof Error ? err.message : t('invitePage.redeemFailed'));
} finally { } finally {
setRedeeming(false); setRedeeming(false);
} }
}, [code, user, onLicenseGranted]); }, [code, user, onLicenseGranted, t]);
if (screenshotMode && loading) { if (screenshotMode && loading) {
return ( return (

View file

@ -262,7 +262,7 @@ function SocialVideoCard({
onPause={() => setIsPlaying(false)} onPause={() => setIsPlaying(false)}
onEnded={() => setIsPlaying(false)} onEnded={() => setIsPlaying(false)}
> >
<track kind="captions" srcLang="en" label="Captions" src={`/video/${slug}.vtt`} /> <track kind="captions" srcLang="en" label={t('common.captions')} src={`/video/${slug}.vtt`} />
</video> </video>
{!isPlaying && ( {!isPlaying && (
<div className="pointer-events-none absolute inset-0 flex items-center justify-center bg-navy-950/15 transition-colors"> <div className="pointer-events-none absolute inset-0 flex items-center justify-center bg-navy-950/15 transition-colors">

View file

@ -1,15 +1,8 @@
import { import { useCallback, useMemo, useState, type MutableRefObject, type ReactNode } from 'react';
useCallback,
useEffect,
useMemo,
useRef,
useState,
type MutableRefObject,
type ReactNode,
} from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server'; import { ts } from '../../i18n/server';
import type { import type {
CrimeRecordsResponse,
FeatureFilters, FeatureFilters,
FeatureGroup, FeatureGroup,
FeatureMeta, FeatureMeta,
@ -39,12 +32,14 @@ import {
} from '../../lib/consts'; } from '../../lib/consts';
import { useNearbyStations } from '../../hooks/useNearbyStations'; import { useNearbyStations } from '../../hooks/useNearbyStations';
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop'; import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
import { useRevealOnExpand } from '../../hooks/useRevealOnExpand';
import { DualHistogram, LoadingSkeleton } from './DualHistogram'; import { DualHistogram, LoadingSkeleton } from './DualHistogram';
import EnumBarChart from './EnumBarChart'; import EnumBarChart from './EnumBarChart';
import StackedBarChart from './StackedBarChart'; import StackedBarChart from './StackedBarChart';
import StackedEnumChart from './StackedEnumChart'; import StackedEnumChart from './StackedEnumChart';
import PriceHistoryChart from './PriceHistoryChart'; import PriceHistoryChart from './PriceHistoryChart';
import CrimeYearChart from './CrimeYearChart'; import CrimeYearChart from './CrimeYearChart';
import CrimeGroupBody from './CrimeGroupBody';
import NumberLine, { type NumberLinePoint } from './NumberLine'; import NumberLine, { type NumberLinePoint } from './NumberLine';
import ExternalSearchLinks from './ExternalSearchLinks'; import ExternalSearchLinks from './ExternalSearchLinks';
import { InfoIcon, TransitIcon } from '../ui/icons'; import { InfoIcon, TransitIcon } from '../ui/icons';
@ -72,6 +67,7 @@ interface AreaPaneProps {
shareCode?: string; shareCode?: string;
isGroupExpanded: (name: string) => boolean; isGroupExpanded: (name: string) => boolean;
onToggleGroup: (name: string) => void; onToggleGroup: (name: string) => void;
onLoadCrimeRecords?: (offset: number) => Promise<CrimeRecordsResponse>;
scrollTopRef?: MutableRefObject<number>; scrollTopRef?: MutableRefObject<number>;
scrollRestoreKey?: string | null; scrollRestoreKey?: string | null;
scrollSaveDisabled?: boolean; scrollSaveDisabled?: boolean;
@ -242,6 +238,7 @@ export default function AreaPane({
shareCode, shareCode,
isGroupExpanded, isGroupExpanded,
onToggleGroup, onToggleGroup,
onLoadCrimeRecords,
scrollTopRef, scrollTopRef,
scrollRestoreKey, scrollRestoreKey,
scrollSaveDisabled, scrollSaveDisabled,
@ -297,61 +294,22 @@ export default function AreaPane({
// When the user expands a group, scroll the bottom of the newly opened content // When the user expands a group, scroll the bottom of the newly opened content
// into view so the whole group is revealed without manual scrolling. The group // into view so the whole group is revealed without manual scrolling. The group
// header stays pinned at the top via its sticky positioning. // header stays pinned at the top via its sticky positioning.
const scrollContainerRef = useRef<HTMLDivElement | null>(null); const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
const setScrollNode = useCallback( const setScrollNode = useCallback(
(node: HTMLDivElement | null) => { (node: HTMLDivElement | null) => {
scrollContainerRef.current = node; setContainer(node);
scrollRef(node); scrollRef(node);
}, },
[scrollRef] [setContainer, scrollRef]
); );
const groupRefs = useRef<Map<string, HTMLDivElement>>(new Map());
// A fresh object each toggle so re-expanding the same group still re-triggers.
const [groupToReveal, setGroupToReveal] = useState<{ name: string } | null>(null);
const handleToggleGroup = useCallback( const handleToggleGroup = useCallback(
(name: string) => { (name: string) => {
const willExpand = !isGroupExpanded(name); const willExpand = !isGroupExpanded(name);
onToggleGroup(name); onToggleGroup(name);
setGroupToReveal(willExpand ? { name } : null); revealGroupOnToggle(name, willExpand);
}, },
[isGroupExpanded, onToggleGroup] [isGroupExpanded, onToggleGroup, revealGroupOnToggle]
); );
useEffect(() => {
if (!groupToReveal) return;
const el = groupRefs.current.get(groupToReveal.name);
const container = scrollContainerRef.current;
if (!el || !container) return;
// Scroll down just enough to bring the bottom of the opened group into view.
// For groups taller than the pane this scrolls past the header, but it stays
// visible via its sticky positioning. Never scroll up (group already in view).
const alignBottom = () => {
const delta = el.getBoundingClientRect().bottom - container.getBoundingClientRect().bottom;
if (delta <= 1) return;
container.scrollTo({ top: container.scrollTop + delta });
};
alignBottom();
// The group's charts and async data (e.g. nearby stations) can grow its
// height after this first pass, so keep the bottom pinned as it settles.
// Stop once the user scrolls or after a short grace period so we never fight
// a deliberate scroll.
let timer = 0;
const observer = new ResizeObserver(alignBottom);
const stop = () => {
observer.disconnect();
container.removeEventListener('wheel', stop);
container.removeEventListener('touchmove', stop);
window.clearTimeout(timer);
};
observer.observe(el);
container.addEventListener('wheel', stop, { passive: true });
container.addEventListener('touchmove', stop, { passive: true });
timer = window.setTimeout(stop, 1500);
return stop;
}, [groupToReveal]);
const numericByName = useMemo(() => { const numericByName = useMemo(() => {
if (!stats) return new Map(); if (!stats) return new Map();
@ -363,26 +321,21 @@ export default function AreaPane({
return new Map(stats.enum_features.map((feature) => [feature.name, feature])); return new Map(stats.enum_features.map((feature) => [feature.name, feature]));
}, [stats]); }, [stats]);
// Crime-by-year series is keyed in the API by the bare crime type (e.g. "Burglary"). // Crime-by-year series, keyed by the bare crime type (e.g. "Burglary").
// We also index by the configured feature name (with " (avg/yr)" suffix) so the const crimeByYearByType = useMemo(() => {
// metric-row renderer can look it up using the feature name it already has.
const crimeByYearByFeatureName = useMemo(() => {
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_by_year']>[number]>(); const map = new Map<string, NonNullable<HexagonStatsResponse['crime_by_year']>[number]>();
for (const entry of stats?.crime_by_year ?? []) { for (const entry of stats?.crime_by_year ?? []) {
map.set(entry.name, entry); map.set(entry.name, entry);
map.set(`${entry.name} (avg/yr)`, entry);
} }
return map; return map;
}, [stats]); }, [stats]);
// Per-crime-type outcode/sector averages, keyed by both the bare crime type // Per-rate-feature outcode/sector averages, keyed by the FULL rate-feature name
// and the " (avg/yr)" feature name (same convention as crimeByYearByFeatureName) // (e.g. "Burglary (/yr, 7y)") the comparison is computed against.
// so renderers can look them up by the feature name they already hold.
const crimeAreaAvgByName = useMemo(() => { const crimeAreaAvgByName = useMemo(() => {
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_area_averages']>[number]>(); const map = new Map<string, NonNullable<HexagonStatsResponse['crime_area_averages']>[number]>();
for (const entry of stats?.crime_area_averages ?? []) { for (const entry of stats?.crime_area_averages ?? []) {
map.set(entry.name, entry); map.set(entry.name, entry);
map.set(`${entry.name} (avg/yr)`, entry);
} }
return map; return map;
}, [stats]); }, [stats]);
@ -440,7 +393,11 @@ export default function AreaPane({
<> <>
<div className="relative flex h-full flex-col"> <div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && stats != null} /> <IndeterminateProgressBar show={loading && stats != null} />
<div ref={setScrollNode} onScroll={onScroll} className="flex-1 overflow-y-auto"> <div
ref={setScrollNode}
onScroll={onScroll}
className="flex-1 overflow-y-auto pb-[env(safe-area-inset-bottom)]"
>
<div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950"> <div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950">
<div className="space-y-3 p-3"> <div className="space-y-3 p-3">
<div className="flex items-start justify-between gap-3"> <div className="flex items-start justify-between gap-3">
@ -615,13 +572,7 @@ export default function AreaPane({
); );
return ( return (
<div <div key={group.name} ref={registerGroup(group.name)}>
key={group.name}
ref={(el) => {
if (el) groupRefs.current.set(group.name, el);
else groupRefs.current.delete(group.name);
}}
>
<CollapsibleGroupHeader <CollapsibleGroupHeader
name={group.name} name={group.name}
expanded={expanded} expanded={expanded}
@ -631,6 +582,19 @@ export default function AreaPane({
{expanded && ( {expanded && (
<div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800"> <div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800">
{showNearbyStations && <NearbyStationsCard location={hexagonLocation} />} {showNearbyStations && <NearbyStationsCard location={hexagonLocation} />}
{group.name === 'Crime' ? (
<CrimeGroupBody
stats={stats}
numericByName={numericByName}
crimeAreaAvgByName={crimeAreaAvgByName}
crimeByYearByType={crimeByYearByType}
globalFeatureByName={globalFeatureByName}
onShowInfo={setInfoFeature}
onLoadCrimeRecords={onLoadCrimeRecords}
selectionKey={`${isPostcode ? 'postcode' : 'hexagon'}:${hexagonId}`}
/>
) : (
<>
{stackedCharts?.map((chart) => { {stackedCharts?.map((chart) => {
const segments = chart.components const segments = chart.components
.map((name) => ({ .map((name) => ({
@ -651,7 +615,7 @@ export default function AreaPane({
? aggregateStats.mean ? aggregateStats.mean
: displaySegments.reduce((sum, s) => sum + s.value, 0); : displaySegments.reduce((sum, s) => sum + s.value, 0);
// Use rateFeature (e.g. per-1k) for display if available // Use rateFeature (e.g. a percentage) for display if available
const rateStats = chart.rateFeature const rateStats = chart.rateFeature
? numericByName.get(chart.rateFeature) ? numericByName.get(chart.rateFeature)
: undefined; : undefined;
@ -715,7 +679,7 @@ export default function AreaPane({
if (total === 0) return null; if (total === 0) return null;
const crimeSeries = chart.feature const crimeSeries = chart.feature
? crimeByYearByFeatureName.get(chart.feature) ? crimeByYearByType.get(chart.feature)
: undefined; : undefined;
return ( return (
@ -800,7 +764,7 @@ export default function AreaPane({
const globalMean = globalHistogram const globalMean = globalHistogram
? calculateHistogramMean(globalHistogram) ? calculateHistogramMean(globalHistogram)
: undefined; : undefined;
const crimeSeries = crimeByYearByFeatureName.get(feature.name); const crimeSeries = crimeByYearByType.get(feature.name);
const crimeAreaAvg = crimeAreaAvgByName.get(feature.name); const crimeAreaAvg = crimeAreaAvgByName.get(feature.name);
// National avg is shown for every metric here as a // National avg is shown for every metric here as a
// tooltip; for crime metrics the outcode and sector // tooltip; for crime metrics the outcode and sector
@ -992,6 +956,8 @@ export default function AreaPane({
</div> </div>
); );
})} })}
</>
)}
</div> </div>
)} )}
</div> </div>

View file

@ -0,0 +1,204 @@
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server';
import type {
CrimeAreaAverage,
CrimeRecordsResponse,
CrimeYearStats,
FeatureMeta,
HexagonStatsResponse,
NumericFeatureStats,
} from '../../types';
import { STACKED_GROUPS, STACKED_SEGMENT_COLORS } from '../../lib/consts';
import { formatValue, calculateHistogramMean } from '../../lib/format';
import { FeatureLabel } from '../ui/FeatureLabel';
import StackedBarChart from './StackedBarChart';
import NumberLine, { type NumberLinePoint } from './NumberLine';
import CrimeYearChart from './CrimeYearChart';
import CrimeRecordsSection from './CrimeRecordsSection';
type CrimeWindow = '7y' | '2y';
/** Strip the " (/yr, 7y|2y)" suffix to the bare crime type. */
function bareType(featureName: string): string {
return featureName.replace(/ \(\/yr, \dy\)$/, '');
}
/** Re-target a crime-feature name at the chosen averaging window. */
function toWindow(featureName: string, window: CrimeWindow): string {
return featureName.replace(/(\/yr, )\dy/, `$1${window}`);
}
/**
* Segment colours keyed by the BARE crime type. The stacked bar then labels each
* segment with the clean crime name (e.g. "Burglary") and keeps one stable colour
* regardless of whether the 7-year or 2-year window is selected.
*/
const SEGMENT_COLOR_BY_BARE: Record<string, string> = Object.fromEntries(
Object.entries(STACKED_SEGMENT_COLORS).map(([name, color]) => [bareType(name), color])
);
interface CrimeGroupBodyProps {
stats: HexagonStatsResponse;
numericByName: Map<string, NumericFeatureStats>;
/** Keyed by the FULL crime-feature name (e.g. "Burglary (/yr, 7y)"). */
crimeAreaAvgByName: Map<string, CrimeAreaAverage>;
/** Keyed by the bare crime type (e.g. "Burglary"). */
crimeByYearByType: Map<string, CrimeYearStats>;
globalFeatureByName: Map<string, FeatureMeta>;
onShowInfo: (feature: FeatureMeta) => void;
onLoadCrimeRecords?: (offset: number) => Promise<CrimeRecordsResponse>;
selectionKey: string | null;
}
export default function CrimeGroupBody({
stats,
numericByName,
crimeAreaAvgByName,
crimeByYearByType,
globalFeatureByName,
onShowInfo,
onLoadCrimeRecords,
selectionKey,
}: CrimeGroupBodyProps) {
const { t } = useTranslation();
// One selector drives every value and comparison in this group.
const [crimeWindow, setCrimeWindow] = useState<CrimeWindow>('7y');
const fmtCount = (value?: number) => (value == null ? '—' : formatValue(value));
const rollupCards = STACKED_GROUPS.Crime ?? [];
return (
<>
<div className="flex items-center justify-between gap-2 py-2">
<span className="text-[11px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
{t('areaPane.crimeWindowLabel')}
</span>
<div className="grid grid-cols-2 gap-0.5 rounded-md bg-warm-200 p-0.5 dark:bg-navy-800">
{(['7y', '2y'] as const).map((w) => {
const active = crimeWindow === w;
return (
<button
key={w}
type="button"
aria-pressed={active}
onClick={() => setCrimeWindow(w)}
className={`rounded px-3 py-1 text-xs font-medium ${
active
? 'bg-white text-teal-700 shadow-sm dark:bg-navy-700 dark:text-teal-300'
: 'text-warm-600 hover:text-warm-900 dark:text-warm-400 dark:hover:text-warm-100'
}`}
>
{t(w === '7y' ? 'areaPane.crimeWindow7y' : 'areaPane.crimeWindow2y')}
</button>
);
})}
</div>
</div>
{rollupCards.map((card) => {
if (!card.feature) return null;
const bare = bareType(card.feature);
const windowFeature = toWindow(card.feature, crimeWindow);
const rate = numericByName.get(windowFeature)?.mean;
const segments = card.components
.map((name) => ({
name: bareType(name),
value: numericByName.get(toWindow(name, crimeWindow))?.mean ?? 0,
}))
.filter((s) => s.value > 0);
const total = segments.reduce((sum, s) => sum + s.value, 0);
if (rate == null && total === 0) return null;
const displayValue = rate ?? total;
// Info popup / icon stay anchored to the 7-year feature so the metric
// definition is stable across windows; only the numbers change.
const featureMeta = globalFeatureByName.get(card.feature);
const globalMean = featureMeta?.histogram
? calculateHistogramMean(featureMeta.histogram)
: undefined;
const crimeAreaAvg = crimeAreaAvgByName.get(windowFeature);
const nationalAvg = crimeAreaAvg?.national ?? globalMean;
const cardTitle = t('areaPane.crimeCardTitle', { name: ts(card.label) });
const numberLinePoints: NumberLinePoint[] = crimeAreaAvg
? (
[
{ kind: 'area', label: t('areaPane.thisArea'), value: displayValue },
nationalAvg != null
? { kind: 'national', label: t('areaPane.national'), value: nationalAvg }
: null,
crimeAreaAvg.outcode != null
? {
kind: 'outcode',
label: stats.crime_outcode ?? t('areaPane.outcodeAvg'),
value: crimeAreaAvg.outcode,
}
: null,
crimeAreaAvg.sector != null
? {
kind: 'sector',
label: stats.crime_sector ?? t('areaPane.sectorAvg'),
value: crimeAreaAvg.sector,
}
: null,
] as (NumberLinePoint | null)[]
).filter((p): p is NumberLinePoint => p !== null)
: [];
const series = crimeByYearByType.get(bare);
return (
<div key={card.label} className="rounded bg-warm-50 p-2 dark:bg-warm-800">
<div className="mb-1.5 flex items-baseline justify-between gap-2">
{featureMeta ? (
<FeatureLabel
feature={{ ...featureMeta, name: ts(card.label) }}
label={cardTitle}
onShowInfo={onShowInfo}
className="mr-2"
wrap
/>
) : (
<span className="mr-2 min-w-0 break-words text-xs leading-snug text-warm-700 dark:text-warm-300">
{cardTitle}
</span>
)}
<div className="shrink-0 text-right">
<span className="whitespace-nowrap text-xs font-semibold text-teal-700 dark:text-teal-400">
{fmtCount(displayValue)}
</span>
</div>
</div>
{total > 0 && (
<StackedBarChart segments={segments} total={total} colorMap={SEGMENT_COLOR_BY_BARE} />
)}
{numberLinePoints.length >= 2 && (
<div className="mt-2">
<NumberLine points={numberLinePoints} format={formatValue} />
</div>
)}
{series && series.points.length > 1 && (
<div className="mt-2">
<CrimeYearChart
points={series.points}
latestAvailableYear={stats.crime_latest_year}
/>
</div>
)}
</div>
);
})}
{onLoadCrimeRecords && (
<CrimeRecordsSection
selectionKey={selectionKey}
total={stats.crime_total_records}
onLoad={onLoadCrimeRecords}
/>
)}
</>
);
}

View file

@ -0,0 +1,147 @@
import { useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server';
import type { CrimeIncident, CrimeRecordsResponse } from '../../types';
import { isAbortError, logNonAbortError } from '../../lib/api';
function formatMonth(month: string, locale: string): string {
const [year, m] = month.split('-').map(Number);
if (!year || !m) return month;
return new Date(Date.UTC(year, m - 1, 1)).toLocaleDateString(locale, {
year: 'numeric',
month: 'short',
timeZone: 'UTC',
});
}
interface CrimeRecordsSectionProps {
/** Identity of the current selection; changing it resets the list. */
selectionKey: string | null;
/** Total records for the selection (from stats); the section hides when 0. */
total?: number;
onLoad: (offset: number) => Promise<CrimeRecordsResponse>;
}
export default function CrimeRecordsSection({
selectionKey,
total,
onLoad,
}: CrimeRecordsSectionProps) {
const { t, i18n } = useTranslation();
const [open, setOpen] = useState(false);
const [records, setRecords] = useState<CrimeIncident[]>([]);
const [loadedTotal, setLoadedTotal] = useState(0);
const [loading, setLoading] = useState(false);
const [error, setError] = useState(false);
// Bumped whenever the selection changes; in-flight responses for a previous
// selection are ignored when they resolve.
const selectionIdRef = useRef(0);
useEffect(() => {
selectionIdRef.current += 1;
setOpen(false);
setRecords([]);
setLoadedTotal(0);
setLoading(false);
setError(false);
}, [selectionKey]);
if (!total || total <= 0) return null;
const fetchPage = async (offset: number) => {
const myId = selectionIdRef.current;
setLoading(true);
setError(false);
try {
const data = await onLoad(offset);
if (myId !== selectionIdRef.current) return;
setRecords((prev) => (offset === 0 ? data.records : [...prev, ...data.records]));
setLoadedTotal(data.total);
} catch (err) {
if (isAbortError(err)) return;
logNonAbortError('Failed to fetch crime records', err);
if (myId === selectionIdRef.current) setError(true);
} finally {
if (myId === selectionIdRef.current) setLoading(false);
}
};
const handleToggle = () => {
const next = !open;
setOpen(next);
if (next && records.length === 0 && !loading) fetchPage(0);
};
const shownTotal = loadedTotal || total;
const canLoadMore = records.length < loadedTotal;
return (
<div className="py-1.5">
<button
type="button"
onClick={handleToggle}
className="flex w-full items-center justify-between gap-2 rounded px-1 py-1 text-left hover:bg-warm-100 dark:hover:bg-navy-800"
aria-expanded={open}
>
<span className="text-[13px] font-medium text-warm-900 dark:text-warm-100">
{open ? t('areaPane.crimeRecordsHide') : t('areaPane.crimeRecordsToggle')}
</span>
<span className="shrink-0 text-xs font-semibold tabular-nums text-warm-500 dark:text-warm-400">
{t('areaPane.crimeRecordsCount', { count: shownTotal })}
</span>
</button>
{open && (
<div className="mt-1.5">
{error ? (
<div className="py-2 text-sm text-rose-600 dark:text-rose-400">
{t('areaPane.crimeRecordsError')}
</div>
) : records.length === 0 && loading ? (
<div className="flex items-center gap-2 py-2 text-sm text-warm-500 dark:text-warm-400">
<span className="h-3 w-3 animate-spin rounded-full border-2 border-teal-600 border-t-transparent dark:border-teal-400 dark:border-t-transparent" />
{t('areaPane.crimeRecordsLoading')}
</div>
) : records.length === 0 ? (
<div className="py-2 text-sm text-warm-500 dark:text-warm-400">
{t('areaPane.crimeRecordsEmpty')}
</div>
) : (
<>
<ol className="divide-y divide-warm-100 dark:divide-navy-800">
{records.map((record, index) => (
<li key={`${record.month}-${record.lat}-${record.lon}-${index}`} className="py-1.5">
<div className="flex items-baseline justify-between gap-2">
<span className="min-w-0 break-words text-[13px] font-medium text-warm-900 dark:text-warm-100">
{ts(record.type)}
</span>
<span className="shrink-0 text-xs tabular-nums text-warm-500 dark:text-warm-400">
{formatMonth(record.month, i18n.language)}
</span>
</div>
<div className="mt-0.5 text-xs text-warm-500 dark:text-warm-400">
{record.location ?? t('areaPane.crimeNoLocation')}
</div>
<div className="text-xs text-warm-400 dark:text-warm-500">
{record.outcome ?? t('areaPane.crimeOutcomeUnknown')}
</div>
</li>
))}
</ol>
{canLoadMore && (
<button
type="button"
onClick={() => !loading && fetchPage(records.length)}
disabled={loading}
className="mt-2 w-full rounded border border-warm-200 px-2 py-1.5 text-xs font-medium text-warm-700 hover:bg-warm-100 disabled:opacity-50 dark:border-navy-700 dark:text-warm-200 dark:hover:bg-navy-800"
>
{loading ? t('areaPane.crimeRecordsLoading') : t('areaPane.crimeRecordsLoadMore')}
</button>
)}
</>
)}
</div>
)}
</div>
);
}

View file

@ -82,7 +82,12 @@ export default function CrimeYearChart({ points, latestAvailableYear }: CrimeYea
r={1.6} r={1.6}
className="fill-rose-700 dark:fill-rose-300" className="fill-rose-700 dark:fill-rose-300"
> >
<title>{`${p.year}: ${p.count.toFixed(1)}/yr`}</title> <title>
{t('areaPane.crimeYearPointTooltip', {
year: p.year,
rate: p.count.toFixed(1),
})}
</title>
</circle> </circle>
))} ))}
<text <text

View file

@ -40,7 +40,7 @@ export default function ExternalSearchLinks({
const radiusMiles = location.isPostcode const radiusMiles = location.isPostcode
? POSTCODE_RADIUS_MILES ? POSTCODE_RADIUS_MILES
: (H3_RADIUS_MILES[location.resolution] ?? 1); : (H3_RADIUS_MILES[location.resolution] ?? 1);
const label = `${radiusMiles}mi radius`; const label = t('externalSearch.radiusMi', { count: radiusMiles });
if (!urls) return null; if (!urls) return null;

View file

@ -33,6 +33,10 @@ interface FeatureBrowserProps {
onClearOpenInfoFeature?: () => void; onClearOpenInfoFeature?: () => void;
travelTimeEntries: TravelTimeEntry[]; travelTimeEntries: TravelTimeEntry[];
onAddTravelTimeEntry: (mode: TransportMode) => void; onAddTravelTimeEntry: (mode: TransportMode) => void;
/** Registers a group wrapper so an expanded group can be scrolled into view. */
registerGroup?: (name: string) => (node: HTMLDivElement | null) => void;
/** Notifies the reveal-on-expand mechanism when a group is toggled. */
onGroupToggle?: (name: string, willExpand: boolean) => void;
} }
export default function FeatureBrowser({ export default function FeatureBrowser({
@ -46,6 +50,8 @@ export default function FeatureBrowser({
onClearOpenInfoFeature, onClearOpenInfoFeature,
travelTimeEntries: _travelTimeEntries, travelTimeEntries: _travelTimeEntries,
onAddTravelTimeEntry, onAddTravelTimeEntry,
registerGroup,
onGroupToggle,
}: FeatureBrowserProps) { }: FeatureBrowserProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const modes = useTranslatedModes(); const modes = useTranslatedModes();
@ -115,11 +121,15 @@ export default function FeatureBrowser({
{mergedGrouped.map((group) => { {mergedGrouped.map((group) => {
const isExpanded = isSearching || isGroupExpanded(group.name); const isExpanded = isSearching || isGroupExpanded(group.name);
return ( return (
<div key={group.name} className="shrink-0"> <div key={group.name} className="shrink-0" ref={registerGroup?.(group.name)}>
<CollapsibleGroupHeader <CollapsibleGroupHeader
name={group.name} name={group.name}
expanded={isExpanded} expanded={isExpanded}
onToggle={() => toggleGroup(group.name)} onToggle={() => {
const willExpand = !isExpanded;
toggleGroup(group.name);
onGroupToggle?.(group.name, willExpand);
}}
className="px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 sticky top-0 z-30 hover:bg-warm-200 dark:hover:bg-warm-800" className="px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 sticky top-0 z-30 hover:bg-warm-200 dark:hover:bg-warm-800"
> >
<span className="text-xs font-medium text-warm-400 dark:text-warm-500"> <span className="text-xs font-medium text-warm-400 dark:text-warm-500">

View file

@ -21,6 +21,16 @@ import {
isSpecificCrimeFeatureName, isSpecificCrimeFeatureName,
isSpecificCrimeFilterName, isSpecificCrimeFilterName,
} from '../../lib/crime-filter'; } from '../../lib/crime-filter';
import {
CRIME_SEVERITY_FILTER_NAMES,
getCrimeSeverityFeatureName,
getCrimeSeverityFilterMeta,
getCrimeSeverityFilterName,
getDefaultCrimeSeverityFeatureName,
isCrimeSeverityFeatureName,
isCrimeSeverityFilterName,
type CrimeSeverityFilterName,
} from '../../lib/crime-severity-filter';
import { import {
ELECTION_VOTE_SHARE_FILTER_NAME, ELECTION_VOTE_SHARE_FILTER_NAME,
getDefaultElectionVoteShareFeatureName, getDefaultElectionVoteShareFeatureName,
@ -37,7 +47,22 @@ import {
isEthnicityFeatureName, isEthnicityFeatureName,
isEthnicityFilterName, isEthnicityFilterName,
} from '../../lib/ethnicity-filter'; } from '../../lib/ethnicity-filter';
import { isQualificationFeatureName } from '../../lib/qualification-filter'; import {
QUALIFICATIONS_FILTER_NAME,
getDefaultQualificationFeatureName,
getQualificationFeatureName,
getQualificationFilterMeta,
isQualificationFeatureName,
isQualificationFilterName,
} from '../../lib/qualification-filter';
import {
TENURE_FILTER_NAME,
getDefaultTenureFeatureName,
getTenureFeatureName,
getTenureFilterMeta,
isTenureFeatureName,
isTenureFilterName,
} from '../../lib/tenure-filter';
import { import {
SCHOOL_FILTER_NAME, SCHOOL_FILTER_NAME,
getDefaultSchoolFeatureName, getDefaultSchoolFeatureName,
@ -182,6 +207,33 @@ export default memo(function Filters({
[features] [features]
); );
const ethnicityMeta = useMemo(() => getEthnicityFilterMeta(features), [features]); const ethnicityMeta = useMemo(() => getEthnicityFilterMeta(features), [features]);
const defaultQualificationFeatureName = useMemo(
() => getDefaultQualificationFeatureName(features),
[features]
);
const qualificationMeta = useMemo(() => getQualificationFilterMeta(features), [features]);
const defaultTenureFeatureName = useMemo(() => getDefaultTenureFeatureName(features), [features]);
const tenureMeta = useMemo(() => getTenureFilterMeta(features), [features]);
const crimeSeverityMetas = useMemo(
() =>
Object.fromEntries(
CRIME_SEVERITY_FILTER_NAMES.map((filterName) => [
filterName,
getCrimeSeverityFilterMeta(features, filterName),
])
) as Record<CrimeSeverityFilterName, FeatureMeta>,
[features]
);
const defaultCrimeSeverityFeatureNames = useMemo(
() =>
Object.fromEntries(
CRIME_SEVERITY_FILTER_NAMES.map((filterName) => [
filterName,
getDefaultCrimeSeverityFeatureName(features, filterName),
])
) as Record<CrimeSeverityFilterName, string | null>,
[features]
);
const defaultPoiDistanceFeatureName = useMemo( const defaultPoiDistanceFeatureName = useMemo(
() => getDefaultPoiDistanceFeatureName(features), () => getDefaultPoiDistanceFeatureName(features),
[features] [features]
@ -256,6 +308,18 @@ export default memo(function Filters({
return { ...(backendFeature ?? specificCrimeMeta), name, group: 'Crime' }; return { ...(backendFeature ?? specificCrimeMeta), name, group: 'Crime' };
}); });
}, [filters, features, specificCrimeMeta]); }, [filters, features, specificCrimeMeta]);
const crimeSeverityFilterItems = useMemo(() => {
return Object.keys(filters)
.filter(isCrimeSeverityFilterName)
.map((name) => {
const backendName = getCrimeSeverityFeatureName(name);
const filterName = getCrimeSeverityFilterName(name) ?? CRIME_SEVERITY_FILTER_NAMES[0];
const backendFeature = backendName
? features.find((feature) => feature.name === backendName)
: undefined;
return { ...(backendFeature ?? crimeSeverityMetas[filterName]), name, group: 'Crime' };
});
}, [filters, features, crimeSeverityMetas]);
const electionVoteShareFilterItems = useMemo(() => { const electionVoteShareFilterItems = useMemo(() => {
return Object.keys(filters) return Object.keys(filters)
.filter(isElectionVoteShareFilterName) .filter(isElectionVoteShareFilterName)
@ -278,6 +342,28 @@ export default memo(function Filters({
return { ...(backendFeature ?? ethnicityMeta), name, group: 'Neighbours' }; return { ...(backendFeature ?? ethnicityMeta), name, group: 'Neighbours' };
}); });
}, [filters, features, ethnicityMeta]); }, [filters, features, ethnicityMeta]);
const qualificationFilterItems = useMemo(() => {
return Object.keys(filters)
.filter(isQualificationFilterName)
.map((name) => {
const backendName = getQualificationFeatureName(name);
const backendFeature = backendName
? features.find((feature) => feature.name === backendName)
: undefined;
return { ...(backendFeature ?? qualificationMeta), name, group: 'Neighbours' };
});
}, [filters, features, qualificationMeta]);
const tenureFilterItems = useMemo(() => {
return Object.keys(filters)
.filter(isTenureFilterName)
.map((name) => {
const backendName = getTenureFeatureName(name);
const backendFeature = backendName
? features.find((feature) => feature.name === backendName)
: undefined;
return { ...(backendFeature ?? tenureMeta), name, group: 'Neighbours' };
});
}, [filters, features, tenureMeta]);
const poiDistanceFilterItems = useMemo(() => { const poiDistanceFilterItems = useMemo(() => {
return Object.keys(filters) return Object.keys(filters)
.filter(isPoiDistanceFilterName) .filter(isPoiDistanceFilterName)
@ -300,6 +386,8 @@ export default memo(function Filters({
let insertedSpecificCrimeFilter = false; let insertedSpecificCrimeFilter = false;
let insertedElectionVoteShareFilter = false; let insertedElectionVoteShareFilter = false;
let insertedEthnicityFilter = false; let insertedEthnicityFilter = false;
let insertedQualificationFilter = false;
let insertedTenureFilter = false;
const insertedPoiFilters = new Set<PoiFilterName>(); const insertedPoiFilters = new Set<PoiFilterName>();
const maybeInsertPoiFilter = (filterName: PoiFilterName | null) => { const maybeInsertPoiFilter = (filterName: PoiFilterName | null) => {
if ( if (
@ -311,6 +399,17 @@ export default memo(function Filters({
insertedPoiFilters.add(filterName); insertedPoiFilters.add(filterName);
} }
}; };
const insertedCrimeSeverityFilters = new Set<CrimeSeverityFilterName>();
const maybeInsertCrimeSeverityFilter = (filterName: CrimeSeverityFilterName | null) => {
if (
filterName &&
defaultCrimeSeverityFeatureNames[filterName] &&
!insertedCrimeSeverityFilters.has(filterName)
) {
result.push(crimeSeverityMetas[filterName]);
insertedCrimeSeverityFilters.add(filterName);
}
};
for (const feature of features) { for (const feature of features) {
if (feature.group === 'Transport') { if (feature.group === 'Transport') {
@ -330,6 +429,12 @@ export default memo(function Filters({
} }
continue; continue;
} }
// "Serious crime" and "Minor crime" each fold their 7y/2y windows into one
// card with a period toggle (no variant dropdown — a single feature each).
if (isCrimeSeverityFeatureName(feature.name)) {
maybeInsertCrimeSeverityFilter(getCrimeSeverityFilterName(feature.name));
continue;
}
if (isElectionVoteShareFeatureName(feature.name)) { if (isElectionVoteShareFeatureName(feature.name)) {
if (defaultElectionVoteShareFeatureName && !insertedElectionVoteShareFilter) { if (defaultElectionVoteShareFeatureName && !insertedElectionVoteShareFilter) {
result.push(electionVoteShareMeta); result.push(electionVoteShareMeta);
@ -344,14 +449,28 @@ export default memo(function Filters({
} }
continue; continue;
} }
// The seven qualification bands fold into one "Qualifications" filter
// whose dropdown picks a band, rather than seven separate sliders.
if (isQualificationFeatureName(feature.name)) {
if (defaultQualificationFeatureName && !insertedQualificationFilter) {
result.push(qualificationMeta);
insertedQualificationFilter = true;
}
continue;
}
// The three tenure categories fold into one "Tenure" filter
// whose dropdown picks a category, rather than three separate sliders.
if (isTenureFeatureName(feature.name)) {
if (defaultTenureFeatureName && !insertedTenureFilter) {
result.push(tenureMeta);
insertedTenureFilter = true;
}
continue;
}
if (isPoiFilterFeatureName(feature.name)) { if (isPoiFilterFeatureName(feature.name)) {
maybeInsertPoiFilter(getPoiFilterName(feature.name)); maybeInsertPoiFilter(getPoiFilterName(feature.name));
continue; continue;
} }
// Qualification breakdown is display-only (shown as the stacked
// "Qualifications" composition in the area pane), so keep its seven
// component features out of the filter browser.
if (isQualificationFeatureName(feature.name)) continue;
if (!enabledFeatures.has(feature.name)) result.push(feature); if (!enabledFeatures.has(feature.name)) result.push(feature);
} }
@ -363,10 +482,16 @@ export default memo(function Filters({
schoolMeta, schoolMeta,
defaultSpecificCrimeFeatureName, defaultSpecificCrimeFeatureName,
specificCrimeMeta, specificCrimeMeta,
defaultCrimeSeverityFeatureNames,
crimeSeverityMetas,
defaultElectionVoteShareFeatureName, defaultElectionVoteShareFeatureName,
electionVoteShareMeta, electionVoteShareMeta,
defaultEthnicityFeatureName, defaultEthnicityFeatureName,
ethnicityMeta, ethnicityMeta,
defaultQualificationFeatureName,
qualificationMeta,
defaultTenureFeatureName,
tenureMeta,
defaultPoiFilterFeatureNames, defaultPoiFilterFeatureNames,
poiFilterMetas, poiFilterMetas,
]); ]);
@ -376,6 +501,8 @@ export default memo(function Filters({
let insertedSpecificCrimeFilters = false; let insertedSpecificCrimeFilters = false;
let insertedElectionVoteShareFilters = false; let insertedElectionVoteShareFilters = false;
let insertedEthnicityFilters = false; let insertedEthnicityFilters = false;
let insertedQualificationFilters = false;
let insertedTenureFilters = false;
const insertedPoiFilters = new Set<PoiFilterName>(); const insertedPoiFilters = new Set<PoiFilterName>();
const insertPoiFilterItems = (filterName: PoiFilterName | null) => { const insertPoiFilterItems = (filterName: PoiFilterName | null) => {
if (!filterName || insertedPoiFilters.has(filterName)) return; if (!filterName || insertedPoiFilters.has(filterName)) return;
@ -384,6 +511,16 @@ export default memo(function Filters({
); );
insertedPoiFilters.add(filterName); insertedPoiFilters.add(filterName);
}; };
const insertedCrimeSeverityFilters = new Set<CrimeSeverityFilterName>();
const insertCrimeSeverityFilterItems = (filterName: CrimeSeverityFilterName | null) => {
if (!filterName || insertedCrimeSeverityFilters.has(filterName)) return;
result.push(
...crimeSeverityFilterItems.filter(
(item) => getCrimeSeverityFilterName(item.name) === filterName
)
);
insertedCrimeSeverityFilters.add(filterName);
};
for (const feature of features) { for (const feature of features) {
if (feature.group === 'Transport') { if (feature.group === 'Transport') {
@ -403,6 +540,10 @@ export default memo(function Filters({
} }
continue; continue;
} }
if (isCrimeSeverityFeatureName(feature.name)) {
insertCrimeSeverityFilterItems(getCrimeSeverityFilterName(feature.name));
continue;
}
if (isElectionVoteShareFeatureName(feature.name)) { if (isElectionVoteShareFeatureName(feature.name)) {
if (!insertedElectionVoteShareFilters) { if (!insertedElectionVoteShareFilters) {
result.push(...electionVoteShareFilterItems); result.push(...electionVoteShareFilterItems);
@ -417,6 +558,20 @@ export default memo(function Filters({
} }
continue; continue;
} }
if (isQualificationFeatureName(feature.name)) {
if (!insertedQualificationFilters) {
result.push(...qualificationFilterItems);
insertedQualificationFilters = true;
}
continue;
}
if (isTenureFeatureName(feature.name)) {
if (!insertedTenureFilters) {
result.push(...tenureFilterItems);
insertedTenureFilters = true;
}
continue;
}
if (isPoiFilterFeatureName(feature.name)) { if (isPoiFilterFeatureName(feature.name)) {
insertPoiFilterItems(getPoiFilterName(feature.name)); insertPoiFilterItems(getPoiFilterName(feature.name));
continue; continue;
@ -430,8 +585,11 @@ export default memo(function Filters({
enabledFeatures, enabledFeatures,
schoolFilterItems, schoolFilterItems,
specificCrimeFilterItems, specificCrimeFilterItems,
crimeSeverityFilterItems,
electionVoteShareFilterItems, electionVoteShareFilterItems,
ethnicityFilterItems, ethnicityFilterItems,
qualificationFilterItems,
tenureFilterItems,
poiDistanceFilterItems, poiDistanceFilterItems,
]); ]);
@ -460,10 +618,15 @@ export default memo(function Filters({
(name: string): string | null => { (name: string): string | null => {
if (name === SCHOOL_FILTER_NAME) return schoolMeta.group ?? 'Schools'; if (name === SCHOOL_FILTER_NAME) return schoolMeta.group ?? 'Schools';
if (name === SPECIFIC_CRIMES_FILTER_NAME) return specificCrimeMeta.group ?? 'Crime'; if (name === SPECIFIC_CRIMES_FILTER_NAME) return specificCrimeMeta.group ?? 'Crime';
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
return crimeSeverityMetas[name as CrimeSeverityFilterName].group ?? 'Crime';
}
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) { if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
return electionVoteShareMeta.group ?? 'Neighbours'; return electionVoteShareMeta.group ?? 'Neighbours';
} }
if (name === ETHNICITIES_FILTER_NAME) return ethnicityMeta.group ?? 'Neighbours'; if (name === ETHNICITIES_FILTER_NAME) return ethnicityMeta.group ?? 'Neighbours';
if (name === QUALIFICATIONS_FILTER_NAME) return qualificationMeta.group ?? 'Neighbours';
if (name === TENURE_FILTER_NAME) return tenureMeta.group ?? 'Neighbours';
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) { if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
return poiFilterMetas[name as PoiFilterName].group ?? null; return poiFilterMetas[name as PoiFilterName].group ?? null;
} }
@ -472,8 +635,11 @@ export default memo(function Filters({
[ [
electionVoteShareMeta.group, electionVoteShareMeta.group,
ethnicityMeta.group, ethnicityMeta.group,
qualificationMeta.group,
tenureMeta.group,
features, features,
poiFilterMetas, poiFilterMetas,
crimeSeverityMetas,
schoolMeta.group, schoolMeta.group,
specificCrimeMeta.group, specificCrimeMeta.group,
] ]
@ -514,6 +680,28 @@ export default memo(function Filters({
onAddFilter(ETHNICITIES_FILTER_NAME); onAddFilter(ETHNICITIES_FILTER_NAME);
return; return;
} }
if (name === QUALIFICATIONS_FILTER_NAME) {
if (!defaultQualificationFeatureName) return;
queueActiveFilterScroll(
QUALIFICATIONS_FILTER_NAME,
getAddFilterGroupName(QUALIFICATIONS_FILTER_NAME)
);
onAddFilter(QUALIFICATIONS_FILTER_NAME);
return;
}
if (name === TENURE_FILTER_NAME) {
if (!defaultTenureFeatureName) return;
queueActiveFilterScroll(TENURE_FILTER_NAME, getAddFilterGroupName(TENURE_FILTER_NAME));
onAddFilter(TENURE_FILTER_NAME);
return;
}
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
const severityFilterName = name as CrimeSeverityFilterName;
if (!defaultCrimeSeverityFeatureNames[severityFilterName]) return;
queueActiveFilterScroll(severityFilterName, getAddFilterGroupName(severityFilterName));
onAddFilter(severityFilterName);
return;
}
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) { if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
const filterName = name as PoiFilterName; const filterName = name as PoiFilterName;
if (!defaultPoiFilterFeatureNames[filterName]) return; if (!defaultPoiFilterFeatureNames[filterName]) return;
@ -528,8 +716,11 @@ export default memo(function Filters({
[ [
defaultSchoolFeatureName, defaultSchoolFeatureName,
defaultSpecificCrimeFeatureName, defaultSpecificCrimeFeatureName,
defaultCrimeSeverityFeatureNames,
defaultElectionVoteShareFeatureName, defaultElectionVoteShareFeatureName,
defaultEthnicityFeatureName, defaultEthnicityFeatureName,
defaultQualificationFeatureName,
defaultTenureFeatureName,
defaultPoiFilterFeatureNames, defaultPoiFilterFeatureNames,
getAddFilterGroupName, getAddFilterGroupName,
onAddFilter, onAddFilter,
@ -686,8 +877,11 @@ export default memo(function Filters({
...features, ...features,
schoolMeta, schoolMeta,
specificCrimeMeta, specificCrimeMeta,
...Object.values(crimeSeverityMetas),
electionVoteShareMeta, electionVoteShareMeta,
ethnicityMeta, ethnicityMeta,
qualificationMeta,
tenureMeta,
poiDistanceMeta, poiDistanceMeta,
transportDistanceMeta, transportDistanceMeta,
poiCount2KmMeta, poiCount2KmMeta,
@ -696,8 +890,11 @@ export default memo(function Filters({
pinnedFeature={pinnedFeature} pinnedFeature={pinnedFeature}
defaultSchoolFeatureName={defaultSchoolFeatureName} defaultSchoolFeatureName={defaultSchoolFeatureName}
defaultSpecificCrimeFeatureName={defaultSpecificCrimeFeatureName} defaultSpecificCrimeFeatureName={defaultSpecificCrimeFeatureName}
defaultCrimeSeverityFeatureNames={defaultCrimeSeverityFeatureNames}
defaultElectionVoteShareFeatureName={defaultElectionVoteShareFeatureName} defaultElectionVoteShareFeatureName={defaultElectionVoteShareFeatureName}
defaultEthnicityFeatureName={defaultEthnicityFeatureName} defaultEthnicityFeatureName={defaultEthnicityFeatureName}
defaultQualificationFeatureName={defaultQualificationFeatureName}
defaultTenureFeatureName={defaultTenureFeatureName}
defaultPoiFilterFeatureNames={defaultPoiFilterFeatureNames} defaultPoiFilterFeatureNames={defaultPoiFilterFeatureNames}
openInfoFeature={openInfoFeature} openInfoFeature={openInfoFeature}
travelTimeEntries={travelTimeEntries} travelTimeEntries={travelTimeEntries}

View file

@ -5,8 +5,11 @@ import { formatValue } from '../../lib/format';
import { ts } from '../../i18n/server'; import { ts } from '../../i18n/server';
import { SCHOOL_FILTER_NAME, getSchoolBackendFeatureName } from '../../lib/school-filter'; import { SCHOOL_FILTER_NAME, getSchoolBackendFeatureName } from '../../lib/school-filter';
import { getSpecificCrimeFeatureName } from '../../lib/crime-filter'; import { getSpecificCrimeFeatureName } from '../../lib/crime-filter';
import { getCrimeSeverityFeatureName } from '../../lib/crime-severity-filter';
import { getElectionVoteShareFeatureName } from '../../lib/election-filter'; import { getElectionVoteShareFeatureName } from '../../lib/election-filter';
import { getEthnicityFeatureName } from '../../lib/ethnicity-filter'; import { getEthnicityFeatureName } from '../../lib/ethnicity-filter';
import { getQualificationFeatureName } from '../../lib/qualification-filter';
import { getTenureFeatureName } from '../../lib/tenure-filter';
import { import {
POI_DISTANCE_FILTER_NAME, POI_DISTANCE_FILTER_NAME,
getPoiDistanceFeatureName, getPoiDistanceFeatureName,
@ -52,14 +55,20 @@ export default memo(function HoverCard({
for (const name of activeFilterNames.slice(0, 4)) { for (const name of activeFilterNames.slice(0, 4)) {
const schoolBackendName = getSchoolBackendFeatureName(name); const schoolBackendName = getSchoolBackendFeatureName(name);
const specificCrimeFeatureName = getSpecificCrimeFeatureName(name); const specificCrimeFeatureName = getSpecificCrimeFeatureName(name);
const crimeSeverityFeatureName = getCrimeSeverityFeatureName(name);
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name); const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
const ethnicityFeatureName = getEthnicityFeatureName(name); const ethnicityFeatureName = getEthnicityFeatureName(name);
const qualificationFeatureName = getQualificationFeatureName(name);
const tenureFeatureName = getTenureFeatureName(name);
const poiDistanceFeatureName = getPoiDistanceFeatureName(name); const poiDistanceFeatureName = getPoiDistanceFeatureName(name);
const backendName = const backendName =
schoolBackendName ?? schoolBackendName ??
specificCrimeFeatureName ?? specificCrimeFeatureName ??
crimeSeverityFeatureName ??
electionVoteShareFeatureName ?? electionVoteShareFeatureName ??
ethnicityFeatureName ?? ethnicityFeatureName ??
qualificationFeatureName ??
tenureFeatureName ??
poiDistanceFeatureName ?? poiDistanceFeatureName ??
name; name;
const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`]; const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`];

View file

@ -16,6 +16,8 @@ vi.mock('react-i18next', () => ({
if (key === 'travel.noBuses') return 'No buses'; if (key === 'travel.noBuses') return 'No buses';
if (key === 'areaPane.walk') return 'Walk'; if (key === 'areaPane.walk') return 'Walk';
if (key === 'areaPane.cycle') return 'Cycle'; if (key === 'areaPane.cycle') return 'Cycle';
if (key === 'journey.bus') return 'Bus';
if (key === 'journey.lineSuffix') return 'line';
if (key === 'areaPane.viewOnGoogleMaps') return 'View on Google Maps'; if (key === 'areaPane.viewOnGoogleMaps') return 'View on Google Maps';
if (key === 'areaPane.noJourneyData') return 'No journey data'; if (key === 'areaPane.noJourneyData') return 'No journey data';
return key; return key;

View file

@ -1,5 +1,6 @@
import { useState, useEffect, useMemo } from 'react'; import { useState, useEffect, useMemo } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import type { TFunction } from 'i18next';
import type { JourneyLeg } from '../../types'; import type { JourneyLeg } from '../../types';
import { resolveTransitVariant, type TravelTimeEntry } from '../../hooks/useTravelTime'; import { resolveTransitVariant, type TravelTimeEntry } from '../../hooks/useTravelTime';
import { apiUrl, authHeaders, logNonAbortError } from '../../lib/api'; import { apiUrl, authHeaders, logNonAbortError } from '../../lib/api';
@ -106,15 +107,25 @@ function stripId(label: string): string {
return label.replace(/\s+\([A-Za-z0-9]+\)$/, ''); return label.replace(/\s+\([A-Za-z0-9]+\)$/, '');
} }
function getRouteDisplay(mode: string): { label: string; color: string; darkText: boolean } { function getRouteDisplay(
mode: string,
t: TFunction
): { label: string; color: string; darkText: boolean } {
const clean = stripId(mode); const clean = stripId(mode);
const known = ROUTE_COLORS[clean]; const known = ROUTE_COLORS[clean];
if (known) { if (known) {
const label = NON_TUBE_NAMES.has(clean) || clean.includes('line') ? clean : `${clean} line`; const label =
NON_TUBE_NAMES.has(clean) || clean.includes('line')
? clean
: `${clean} ${t('journey.lineSuffix')}`;
return { label, color: known.color, darkText: !!known.darkText }; return { label, color: known.color, darkText: !!known.darkText };
} }
if (/^\d+[A-Za-z]?$/.test(clean.trim())) { if (/^\d+[A-Za-z]?$/.test(clean.trim())) {
return { label: `Bus ${clean}`, color: '#0d9488', darkText: false }; return {
label: `${t('journey.bus')} ${clean}`,
color: '#0d9488',
darkText: false,
};
} }
return { label: clean, color: '#6b7280', darkText: false }; return { label: clean, color: '#6b7280', darkText: false };
} }
@ -203,7 +214,8 @@ function invertLegs(legs: JourneyLeg[]): JourneyLeg[] {
} }
function RouteBadge({ mode }: { mode: string }) { function RouteBadge({ mode }: { mode: string }) {
const { label, color, darkText } = getRouteDisplay(mode); const { t } = useTranslation();
const { label, color, darkText } = getRouteDisplay(mode, t);
return ( return (
<span <span
className="inline-flex items-center text-[10px] font-bold px-1.5 py-px rounded-sm leading-tight tracking-wide" className="inline-flex items-center text-[10px] font-bold px-1.5 py-px rounded-sm leading-tight tracking-wide"
@ -242,7 +254,7 @@ function TimelineLeg({ leg, isLast }: { leg: JourneyLeg; isLast: boolean }) {
); );
} }
const { color } = getRouteDisplay(leg.mode); const { color } = getRouteDisplay(leg.mode, t);
return ( return (
<div className="flex"> <div className="flex">
<div className="flex flex-col items-center w-4 mr-2"> <div className="flex flex-col items-center w-4 mr-2">

View file

@ -19,15 +19,21 @@ function formatListingHeadline(listing: ActualListing, t: TFunction): string | n
export const ListingPopupSingleContent = memo(function ListingPopupSingleContent({ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent({
listing, listing,
clickedUrls,
onOpen,
}: { }: {
listing: ActualListing; listing: ActualListing;
clickedUrls: Set<string>;
onOpen: (url: string) => void;
}) { }) {
const { t } = useTranslation(); const { t } = useTranslation();
const visited = clickedUrls.has(listing.listing_url);
return ( return (
<a <a
href={listing.listing_url} href={listing.listing_url}
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
onClick={() => onOpen(listing.listing_url)}
className="block px-3 py-2" className="block px-3 py-2"
> >
{listing.asking_price != null && ( {listing.asking_price != null && (
@ -57,7 +63,7 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
)} )}
{listing.floor_area_sqm != null && ( {listing.floor_area_sqm != null && (
<div className="text-[11px] text-warm-500 dark:text-warm-400 mt-0.5"> <div className="text-[11px] text-warm-500 dark:text-warm-400 mt-0.5">
{Math.round(listing.floor_area_sqm)} sqm {Math.round(listing.floor_area_sqm)} {t('common.sqm')}
{listing.asking_price_per_sqm != null {listing.asking_price_per_sqm != null
? ` · £${Math.round(listing.asking_price_per_sqm).toLocaleString()}/sqm` ? ` · £${Math.round(listing.asking_price_per_sqm).toLocaleString()}/sqm`
: ''} : ''}
@ -72,8 +78,9 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
))} ))}
</ul> </ul>
)} )}
<div className="mt-1.5 text-[11px] text-teal-600 dark:text-teal-400 font-medium"> <div className="mt-1.5 flex items-center gap-1.5 text-[11px] font-medium">
Open listing {visited && <span className="text-violet-600 dark:text-violet-400"> {t('listing.viewed')}</span>}
<span className="text-teal-600 dark:text-teal-400">{t('listing.openListing')} </span>
</div> </div>
</a> </a>
); );
@ -82,9 +89,13 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
export const ListingClusterPopupContent = memo(function ListingClusterPopupContent({ export const ListingClusterPopupContent = memo(function ListingClusterPopupContent({
count, count,
listings, listings,
clickedUrls,
onOpen,
}: { }: {
count: number; count: number;
listings: ActualListing[]; listings: ActualListing[];
clickedUrls: Set<string>;
onOpen: (url: string) => void;
}) { }) {
const { t } = useTranslation(); const { t } = useTranslation();
const visibleCount = listings.length; const visibleCount = listings.length;
@ -92,32 +103,41 @@ export const ListingClusterPopupContent = memo(function ListingClusterPopupConte
<div> <div>
<div className="border-b border-warm-200 px-3 py-2 dark:border-warm-700"> <div className="border-b border-warm-200 px-3 py-2 dark:border-warm-700">
<div className="text-base font-bold text-red-600 dark:text-red-400"> <div className="text-base font-bold text-red-600 dark:text-red-400">
{count.toLocaleString()} listings {count.toLocaleString()} {t('listing.listings')}
</div> </div>
<div className="text-[11px] text-warm-500 dark:text-warm-400"> <div className="text-[11px] text-warm-500 dark:text-warm-400">
{visibleCount > 0 {visibleCount > 0
? `Showing ${visibleCount.toLocaleString()} of ${count.toLocaleString()}` ? t('listing.showingOf', { visible: visibleCount, count })
: 'Grouped near this map position'} : t('listing.groupedNear')}
</div> </div>
</div> </div>
{visibleCount > 0 && ( {visibleCount > 0 && (
<div className="max-h-80 overflow-y-auto py-1"> <div className="max-h-80 overflow-y-auto py-1">
{listings.map((listing, idx) => { {listings.map((listing, idx) => {
const headline = formatListingHeadline(listing, t); const headline = formatListingHeadline(listing, t);
const visited = clickedUrls.has(listing.listing_url);
return ( return (
<a <a
key={`${listing.listing_url}-${idx}`} key={`${listing.listing_url}-${idx}`}
href={listing.listing_url} href={listing.listing_url}
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
onClick={() => onOpen(listing.listing_url)}
className="block border-b border-warm-100 px-3 py-2 last:border-b-0 hover:bg-warm-50 dark:border-warm-700 dark:hover:bg-warm-700/60" className="block border-b border-warm-100 px-3 py-2 last:border-b-0 hover:bg-warm-50 dark:border-warm-700 dark:hover:bg-warm-700/60"
> >
<div className="flex items-start justify-between gap-3"> <div className="flex items-start justify-between gap-3">
<div className="min-w-0"> <div className="min-w-0">
<div className="text-sm font-semibold text-teal-700 dark:text-teal-300"> <div className="flex items-center gap-1.5">
{listing.asking_price != null <span className="text-sm font-semibold text-teal-700 dark:text-teal-300">
? formatListingPrice(listing.asking_price) {listing.asking_price != null
: 'Listing'} ? formatListingPrice(listing.asking_price)
: t('listing.listing')}
</span>
{visited && (
<span className="shrink-0 text-[10px] font-medium text-violet-600 dark:text-violet-400">
Viewed
</span>
)}
</div> </div>
{headline && ( {headline && (
<div className="mt-0.5 truncate text-xs text-warm-700 dark:text-warm-200"> <div className="mt-0.5 truncate text-xs text-warm-700 dark:text-warm-200">

View file

@ -452,6 +452,8 @@ export default memo(function Map({
visiblePois, visiblePois,
listingPopup, listingPopup,
clearListingPopup, clearListingPopup,
clickedListingUrls,
markListingClicked,
developmentPopup, developmentPopup,
clearDevelopmentPopup, clearDevelopmentPopup,
hoverPosition, hoverPosition,
@ -696,11 +698,17 @@ export default memo(function Map({
<CloseIcon className="w-3 h-3" /> <CloseIcon className="w-3 h-3" />
</button> </button>
{listingPopup.mode === 'single' ? ( {listingPopup.mode === 'single' ? (
<ListingPopupSingleContent listing={listingPopup.listing} /> <ListingPopupSingleContent
listing={listingPopup.listing}
clickedUrls={clickedListingUrls}
onOpen={markListingClicked}
/>
) : ( ) : (
<ListingClusterPopupContent <ListingClusterPopupContent
count={listingPopup.count} count={listingPopup.count}
listings={listingPopup.listings} listings={listingPopup.listings}
clickedUrls={clickedListingUrls}
onOpen={markListingClicked}
/> />
)} )}
</div> </div>

View file

@ -1,6 +1,7 @@
import { Component, Fragment, type ReactNode } from 'react'; import { Component, Fragment, type ReactNode } from 'react';
import * as Sentry from '@sentry/react'; import * as Sentry from '@sentry/react';
import i18n from '../../i18n';
import { MapFallback } from './map-page/Fallbacks'; import { MapFallback } from './map-page/Fallbacks';
/** /**
@ -113,17 +114,19 @@ export class MapErrorBoundary extends Component<MapErrorBoundaryProps, MapErrorB
<div className="flex h-full w-full items-center justify-center bg-warm-100 px-6 text-center dark:bg-navy-950"> <div className="flex h-full w-full items-center justify-center bg-warm-100 px-6 text-center dark:bg-navy-950">
<div> <div>
<h2 className="text-lg font-semibold text-warm-900 dark:text-warm-100"> <h2 className="text-lg font-semibold text-warm-900 dark:text-warm-100">
The map ran into a problem {i18n.t('map.error.heading', { defaultValue: 'The map ran into a problem' })}
</h2> </h2>
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300"> <p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
This can happen when your browser&apos;s graphics context is interrupted. {i18n.t('map.error.body', {
defaultValue: 'This can happen when your browsers graphics context is interrupted.',
})}
</p> </p>
<button <button
type="button" type="button"
onClick={this.handleManualRetry} onClick={this.handleManualRetry}
className="mt-4 rounded-lg bg-teal-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-teal-700 dark:bg-teal-500 dark:hover:bg-teal-400" className="mt-4 rounded-lg bg-teal-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-teal-700 dark:bg-teal-500 dark:hover:bg-teal-400"
> >
Reload the map {i18n.t('map.error.reload', { defaultValue: 'Reload the map' })}
</button> </button>
</div> </div>
</div> </div>

View file

@ -407,6 +407,7 @@ export default function MapPage({
handlePropertiesTabClick, handlePropertiesTabClick,
handleLoadMoreProperties, handleLoadMoreProperties,
handleCloseSelection, handleCloseSelection,
loadCrimeRecords,
selectedPostcodeGeometry, selectedPostcodeGeometry,
handleLocationSearch, handleLocationSearch,
handleCurrentLocationSearch, handleCurrentLocationSearch,
@ -806,6 +807,7 @@ export default function MapPage({
shareCode={shareCode} shareCode={shareCode}
isGroupExpanded={isAreaGroupExpanded} isGroupExpanded={isAreaGroupExpanded}
onToggleGroup={toggleAreaGroup} onToggleGroup={toggleAreaGroup}
onLoadCrimeRecords={loadCrimeRecords}
scrollTopRef={areaPaneScrollTopRef} scrollTopRef={areaPaneScrollTopRef}
scrollRestoreKey={ scrollRestoreKey={
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
@ -823,6 +825,7 @@ export default function MapPage({
hexagonLocation, hexagonLocation,
isAreaGroupExpanded, isAreaGroupExpanded,
loadingAreaStats, loadingAreaStats,
loadCrimeRecords,
selectedHexagon, selectedHexagon,
setAreaStatsUseFilters, setAreaStatsUseFilters,
shareCode, shareCode,

View file

@ -105,7 +105,11 @@ export default function MobileDrawer({
return ( return (
<div <div
data-tutorial="right-pane" data-tutorial="right-pane"
className="pointer-events-none fixed inset-0 z-50 flex flex-col" // Pin to the dynamic viewport (100dvh) anchored at the top instead of
// `inset-0`. On iOS Safari a fixed element with `bottom: 0` is laid out
// against the large viewport, so when the bottom toolbar shows the panel's
// bottom (and the last collapsible group) hides behind it and can't be tapped.
className="pointer-events-none fixed inset-x-0 top-0 z-50 flex h-[100dvh] flex-col"
> >
<div className="h-[10%] shrink-0" aria-hidden="true" /> <div className="h-[10%] shrink-0" aria-hidden="true" />

View file

@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next';
import { BASEMAPS, type BasemapId } from '../../lib/basemaps'; import { BASEMAPS, type BasemapId } from '../../lib/basemaps';
import { OVERLAYS, type OverlayDefinition, type OverlayId } from '../../lib/overlays'; import { OVERLAYS, type OverlayDefinition, type OverlayId } from '../../lib/overlays';
import { CRIME_TYPES, CRIME_TYPE_VALUES } from '../../lib/crime-types'; import { CRIME_TYPES, CRIME_TYPE_VALUES } from '../../lib/crime-types';
import { tDynamic } from '../../i18n';
import { PillToggle } from '../ui/PillToggle'; import { PillToggle } from '../ui/PillToggle';
import { IconButton } from '../ui/IconButton'; import { IconButton } from '../ui/IconButton';
import { Slider } from '../ui/Slider'; import { Slider } from '../ui/Slider';
@ -12,6 +13,18 @@ import { colorOpacityToPercent, normalizeColorOpacity } from '../../lib/color-op
const CRIME_OVERLAY_ID: OverlayId = 'crime-hotspots'; const CRIME_OVERLAY_ID: OverlayId = 'crime-hotspots';
// Maps the kebab-case overlay id to its camelCase i18n key under `overlays.*`.
const OVERLAY_I18N_KEY: Record<OverlayId, string> = {
noise: 'noise',
'crime-hotspots': 'crimeHotspots',
'trees-outside-woodlands': 'treesOutsideWoodlands',
'property-borders': 'propertyBorders',
'new-developments': 'newDevelopments',
};
const overlayLabel = (id: OverlayId) => tDynamic(`overlays.${OVERLAY_I18N_KEY[id]}.label`);
const overlayDetail = (id: OverlayId) => tDynamic(`overlays.${OVERLAY_I18N_KEY[id]}.detail`);
interface OverlayPaneProps { interface OverlayPaneProps {
selectedOverlays: Set<OverlayId>; selectedOverlays: Set<OverlayId>;
onOverlaysChange: (overlays: Set<OverlayId>) => void; onOverlaysChange: (overlays: Set<OverlayId>) => void;
@ -78,7 +91,7 @@ export default function OverlayPane({
<div className="flex-shrink-0 px-3 pt-3 pb-2"> <div className="flex-shrink-0 px-3 pt-3 pb-2">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="text-xs font-semibold text-warm-500 dark:text-warm-400 uppercase tracking-wide"> <span className="text-xs font-semibold text-warm-500 dark:text-warm-400 uppercase tracking-wide">
Overlays {t('overlays.heading')}
</span> </span>
<span className="text-xs text-warm-400 dark:text-warm-500"> <span className="text-xs text-warm-400 dark:text-warm-500">
{selectedOverlays.size}/{OVERLAYS.length} {selectedOverlays.size}/{OVERLAYS.length}
@ -88,13 +101,13 @@ export default function OverlayPane({
onClick={selectNone} onClick={selectNone}
className="rounded border border-warm-300 px-2 py-0.5 text-xs text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700" className="rounded border border-warm-300 px-2 py-0.5 text-xs text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
> >
None {t('common.none')}
</button> </button>
{onClose && ( {onClose && (
<button <button
onClick={onClose} onClick={onClose}
className="ml-1 p-0.5 text-warm-400 hover:text-warm-700 dark:hover:text-warm-300" className="ml-1 p-0.5 text-warm-400 hover:text-warm-700 dark:hover:text-warm-300"
title="Close" title={t('common.close')}
> >
<CloseIcon className="h-4 w-4" /> <CloseIcon className="h-4 w-4" />
</button> </button>
@ -106,8 +119,7 @@ export default function OverlayPane({
role="alert" role="alert"
className="mt-2 rounded border border-amber-300 bg-amber-50 px-2 py-1.5 text-xs text-amber-800 dark:border-amber-700/60 dark:bg-amber-900/30 dark:text-amber-200" className="mt-2 rounded border border-amber-300 bg-amber-50 px-2 py-1.5 text-xs text-amber-800 dark:border-amber-700/60 dark:bg-amber-900/30 dark:text-amber-200"
> >
Zoom in further to see the selected{' '} {t('overlays.zoomWarning', { count: selectedOverlays.size })}
{selectedOverlays.size === 1 ? 'overlay' : 'overlays'}.
</div> </div>
)} )}
</div> </div>
@ -115,13 +127,13 @@ export default function OverlayPane({
<div className="min-h-0 space-y-4 overflow-y-auto overscroll-contain border-t border-warm-200 px-3 py-3 dark:border-warm-700"> <div className="min-h-0 space-y-4 overflow-y-auto overscroll-contain border-t border-warm-200 px-3 py-3 dark:border-warm-700">
<div> <div>
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500"> <div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
Base map {t('overlays.baseMap')}
</div> </div>
<div className="flex flex-wrap gap-1.5"> <div className="flex flex-wrap gap-1.5">
{BASEMAPS.map((option) => ( {BASEMAPS.map((option) => (
<PillToggle <PillToggle
key={option.id} key={option.id}
label={option.label} label={tDynamic(`map.basemap.${option.id}`)}
active={basemap === option.id} active={basemap === option.id}
onClick={() => onBasemapChange(option.id)} onClick={() => onBasemapChange(option.id)}
size="sm" size="sm"
@ -133,7 +145,7 @@ export default function OverlayPane({
<div> <div>
<div className="mb-2 flex items-center justify-between gap-2"> <div className="mb-2 flex items-center justify-between gap-2">
<span className="text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500"> <span className="text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
Colour opacity {t('overlays.colourOpacity')}
</span> </span>
<span className="text-[10px] font-medium tabular-nums text-warm-500 dark:text-warm-400"> <span className="text-[10px] font-medium tabular-nums text-warm-500 dark:text-warm-400">
{colorOpacityPercent}% {colorOpacityPercent}%
@ -145,27 +157,27 @@ export default function OverlayPane({
step={5} step={5}
value={[colorOpacityPercent]} value={[colorOpacityPercent]}
onValueChange={handleColorOpacityChange} onValueChange={handleColorOpacityChange}
aria-label="Colour opacity" aria-label={t('overlays.colourOpacity')}
/> />
</div> </div>
<div> <div>
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500"> <div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
Data overlays {t('overlays.dataOverlays')}
</div> </div>
<div className="flex flex-wrap gap-1.5"> <div className="flex flex-wrap gap-1.5">
{OVERLAYS.map((overlay) => ( {OVERLAYS.map((overlay) => (
<div key={overlay.id} className="inline-flex items-center gap-0.5"> <div key={overlay.id} className="inline-flex items-center gap-0.5">
<PillToggle <PillToggle
label={overlay.label} label={overlayLabel(overlay.id)}
active={selectedOverlays.has(overlay.id)} active={selectedOverlays.has(overlay.id)}
onClick={() => toggleOverlay(overlay.id)} onClick={() => toggleOverlay(overlay.id)}
size="sm" size="sm"
/> />
<IconButton <IconButton
onClick={() => setInfoOverlay(overlay)} onClick={() => setInfoOverlay(overlay)}
title={`About ${overlay.label}`} title={t('overlays.about', { name: overlayLabel(overlay.id) })}
ariaLabel={`About ${overlay.label}`} ariaLabel={t('overlays.about', { name: overlayLabel(overlay.id) })}
> >
<InfoIcon className="h-3.5 w-3.5" /> <InfoIcon className="h-3.5 w-3.5" />
</IconButton> </IconButton>
@ -188,13 +200,13 @@ export default function OverlayPane({
onClick={selectAllCrimeTypes} onClick={selectAllCrimeTypes}
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700" className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
> >
All {t('common.all')}
</button> </button>
<button <button
onClick={selectNoCrimeTypes} onClick={selectNoCrimeTypes}
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700" className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
> >
None {t('common.none')}
</button> </button>
</div> </div>
</div> </div>
@ -210,7 +222,7 @@ export default function OverlayPane({
onChange={() => toggleCrimeType(crime.value)} onChange={() => toggleCrimeType(crime.value)}
className="h-3.5 w-3.5 shrink-0 rounded border-warm-300 text-amber-600 focus:ring-1 focus:ring-amber-500 dark:border-warm-600 dark:bg-warm-800" className="h-3.5 w-3.5 shrink-0 rounded border-warm-300 text-amber-600 focus:ring-1 focus:ring-amber-500 dark:border-warm-600 dark:bg-warm-800"
/> />
<span>{crime.label}</span> <span>{tDynamic(crime.labelKey)}</span>
</label> </label>
))} ))}
</div> </div>
@ -219,9 +231,9 @@ export default function OverlayPane({
</div> </div>
{infoOverlay && ( {infoOverlay && (
<InfoPopup title={infoOverlay.label} onClose={() => setInfoOverlay(null)}> <InfoPopup title={overlayLabel(infoOverlay.id)} onClose={() => setInfoOverlay(null)}>
<p className="text-sm text-warm-700 dark:text-warm-300 leading-relaxed"> <p className="text-sm text-warm-700 dark:text-warm-300 leading-relaxed">
{infoOverlay.detail} {overlayDetail(infoOverlay.id)}
</p> </p>
</InfoPopup> </InfoPopup>
)} )}

View file

@ -1,4 +1,6 @@
import { memo } from 'react'; import { memo } from 'react';
import { useTranslation } from 'react-i18next';
import type { TFunction } from 'i18next';
import type { SchoolMetadata } from '../../types'; import type { SchoolMetadata } from '../../types';
import { POI_GROUP_COLORS } from '../../lib/consts'; import { POI_GROUP_COLORS } from '../../lib/consts';
@ -32,7 +34,7 @@ function normalizeSchoolWebsiteUrl(raw: string): string | null {
return null; return null;
} }
function renderSchoolMetadata(school: SchoolMetadata) { function renderSchoolMetadata(school: SchoolMetadata, t: TFunction) {
// First line collects the headline classification (phase, type, religious // First line collects the headline classification (phase, type, religious
// character) so the popup is scannable even when most fields are absent. // character) so the popup is scannable even when most fields are absent.
const headline: string[] = []; const headline: string[] = [];
@ -41,11 +43,11 @@ function renderSchoolMetadata(school: SchoolMetadata) {
const pupilsLine = const pupilsLine =
school.pupils !== undefined && school.capacity !== undefined school.pupils !== undefined && school.capacity !== undefined
? `${school.pupils.toLocaleString()} / ${school.capacity.toLocaleString()} pupils` ? `${school.pupils.toLocaleString()} / ${school.capacity.toLocaleString()} ${t('poiPopup.school.pupils')}`
: school.pupils !== undefined : school.pupils !== undefined
? `${school.pupils.toLocaleString()} pupils` ? `${school.pupils.toLocaleString()} ${t('poiPopup.school.pupils')}`
: school.capacity !== undefined : school.capacity !== undefined
? `Capacity ${school.capacity.toLocaleString()}` ? `${t('poiPopup.school.capacity')} ${school.capacity.toLocaleString()}`
: null; : null;
const websiteUrl = school.website ? normalizeSchoolWebsiteUrl(school.website) : null; const websiteUrl = school.website ? normalizeSchoolWebsiteUrl(school.website) : null;
@ -54,69 +56,69 @@ function renderSchoolMetadata(school: SchoolMetadata) {
<dl className="mt-2 grid grid-cols-[auto_1fr] gap-x-2 gap-y-0.5 text-xs text-warm-600 dark:text-warm-300"> <dl className="mt-2 grid grid-cols-[auto_1fr] gap-x-2 gap-y-0.5 text-xs text-warm-600 dark:text-warm-300">
{headline.length > 0 && ( {headline.length > 0 && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Type</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.type')}</dt>
<dd className="dark:text-warm-200">{headline.join(' · ')}</dd> <dd className="dark:text-warm-200">{headline.join(' · ')}</dd>
</> </>
)} )}
{school.age_range && ( {school.age_range && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Ages</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.ages')}</dt>
<dd className="dark:text-warm-200">{school.age_range}</dd> <dd className="dark:text-warm-200">{school.age_range}</dd>
</> </>
)} )}
{school.gender && school.gender !== 'Mixed' && ( {school.gender && school.gender !== 'Mixed' && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Gender</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.gender')}</dt>
<dd className="dark:text-warm-200">{school.gender}</dd> <dd className="dark:text-warm-200">{school.gender}</dd>
</> </>
)} )}
{pupilsLine && ( {pupilsLine && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Pupils</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.pupilsLabel')}</dt>
<dd className="dark:text-warm-200">{pupilsLine}</dd> <dd className="dark:text-warm-200">{pupilsLine}</dd>
</> </>
)} )}
{school.fsm_percent !== undefined && ( {school.fsm_percent !== undefined && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Free meal</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.freeMeal')}</dt>
<dd className="dark:text-warm-200">{school.fsm_percent.toFixed(1)}%</dd> <dd className="dark:text-warm-200">{school.fsm_percent.toFixed(1)}%</dd>
</> </>
)} )}
{school.ofsted_rating && ( {school.ofsted_rating && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Ofsted</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.ofsted')}</dt>
<dd className="dark:text-warm-200">{school.ofsted_rating}</dd> <dd className="dark:text-warm-200">{school.ofsted_rating}</dd>
</> </>
)} )}
{school.sixth_form === 'Has a sixth form' && ( {school.sixth_form === 'Has a sixth form' && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Sixth form</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.sixthForm')}</dt>
<dd className="dark:text-warm-200">Yes</dd> <dd className="dark:text-warm-200">{t('common.yes')}</dd>
</> </>
)} )}
{school.religious_character && {school.religious_character &&
school.religious_character !== 'Does not apply' && school.religious_character !== 'Does not apply' &&
school.religious_character !== 'None' && ( school.religious_character !== 'None' && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Religion</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.religion')}</dt>
<dd className="dark:text-warm-200">{school.religious_character}</dd> <dd className="dark:text-warm-200">{school.religious_character}</dd>
</> </>
)} )}
{school.admissions_policy && ( {school.admissions_policy && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Admissions</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.admissions')}</dt>
<dd className="dark:text-warm-200">{school.admissions_policy}</dd> <dd className="dark:text-warm-200">{school.admissions_policy}</dd>
</> </>
)} )}
{school.trust && ( {school.trust && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Trust</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.trust')}</dt>
<dd className="dark:text-warm-200">{school.trust}</dd> <dd className="dark:text-warm-200">{school.trust}</dd>
</> </>
)} )}
{(school.address || school.postcode) && ( {(school.address || school.postcode) && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Address</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.address')}</dt>
<dd className="dark:text-warm-200"> <dd className="dark:text-warm-200">
{[school.address, school.postcode].filter(Boolean).join(', ')} {[school.address, school.postcode].filter(Boolean).join(', ')}
</dd> </dd>
@ -124,19 +126,19 @@ function renderSchoolMetadata(school: SchoolMetadata) {
)} )}
{school.local_authority && ( {school.local_authority && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">LA</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.localAuthority')}</dt>
<dd className="dark:text-warm-200">{school.local_authority}</dd> <dd className="dark:text-warm-200">{school.local_authority}</dd>
</> </>
)} )}
{school.head_name && ( {school.head_name && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Head</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.head')}</dt>
<dd className="dark:text-warm-200">{school.head_name}</dd> <dd className="dark:text-warm-200">{school.head_name}</dd>
</> </>
)} )}
{websiteUrl && ( {websiteUrl && (
<> <>
<dt className="text-warm-500 dark:text-warm-400">Website</dt> <dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.website')}</dt>
<dd className="truncate"> <dd className="truncate">
<a <a
href={websiteUrl} href={websiteUrl}
@ -158,6 +160,7 @@ export const PoiPopupCardContent = memo(function PoiPopupCardContent({
}: { }: {
poi: PoiPopupCardData; poi: PoiPopupCardData;
}) { }) {
const { t } = useTranslation();
return ( return (
<div className="px-3 py-2 max-w-[280px]"> <div className="px-3 py-2 max-w-[280px]">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
@ -182,7 +185,7 @@ export const PoiPopupCardContent = memo(function PoiPopupCardContent({
</div> </div>
</div> </div>
</div> </div>
{poi.school && renderSchoolMetadata(poi.school)} {poi.school && renderSchoolMetadata(poi.school, t)}
</div> </div>
); );
}); });

View file

@ -79,7 +79,11 @@ export function PropertiesPane({
return ( return (
<div className="relative flex h-full flex-col"> <div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && properties.length > 0} /> <IndeterminateProgressBar show={loading && properties.length > 0} />
<div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto"> <div
ref={scrollRef}
onScroll={onScroll}
className="flex-1 overflow-y-auto pb-[env(safe-area-inset-bottom)]"
>
{showInfo && ( {showInfo && (
<InfoPopup <InfoPopup
title={t('propertyCard.propertyData')} title={t('propertyCard.propertyData')}

View file

@ -18,6 +18,7 @@ interface StackedBarChartProps {
/** Strip common suffixes/prefixes to produce short legend labels */ /** Strip common suffixes/prefixes to produce short legend labels */
function shortenLabel(name: string): string { function shortenLabel(name: string): string {
return name return name
.replace(/ \(\/yr, \d+y\)$/, '')
.replace(' (avg/yr)', '') .replace(' (avg/yr)', '')
.replace(/^% /, '') .replace(/^% /, '')
.replace('and sexual offences', '') .replace('and sexual offences', '')

View file

@ -3,12 +3,32 @@ import { useMemo } from 'react';
import type { FeatureFilters, FeatureMeta } from '../../../types'; import type { FeatureFilters, FeatureMeta } from '../../../types';
import type { PercentileScale } from '../../../lib/format'; import type { PercentileScale } from '../../../lib/format';
import { groupFeaturesByCategory } from '../../../lib/features'; import { groupFeaturesByCategory } from '../../../lib/features';
import { getSpecificCrimeFeatureName, isSpecificCrimeFilterName } from '../../../lib/crime-filter'; import {
SPECIFIC_CRIME_VARIANT_CONFIG,
getSpecificCrimeFeatureName,
isSpecificCrimeFilterName,
} from '../../../lib/crime-filter';
import {
getCrimeSeverityFeatureName,
getCrimeSeverityFilterName,
getCrimeSeverityVariantConfig,
isCrimeSeverityFilterName,
} from '../../../lib/crime-severity-filter';
import { import {
getElectionVoteShareFeatureName, getElectionVoteShareFeatureName,
isElectionVoteShareFilterName, isElectionVoteShareFilterName,
} from '../../../lib/election-filter'; } from '../../../lib/election-filter';
import { getEthnicityFeatureName, isEthnicityFilterName } from '../../../lib/ethnicity-filter'; import { getEthnicityFeatureName, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
import {
QUALIFICATION_VARIANT_CONFIG,
getQualificationFeatureName,
isQualificationFilterName,
} from '../../../lib/qualification-filter';
import {
TENURE_VARIANT_CONFIG,
getTenureFeatureName,
isTenureFilterName,
} from '../../../lib/tenure-filter';
import { getSchoolBackendFeatureName, isSchoolFilterName } from '../../../lib/school-filter'; import { getSchoolBackendFeatureName, isSchoolFilterName } from '../../../lib/school-filter';
import { import {
getPoiDistanceFeatureName, getPoiDistanceFeatureName,
@ -18,7 +38,7 @@ import type { TravelTimeEntry } from '../../../hooks/useTravelTime';
import { EthnicityFilterCard } from './EthnicityFilterCard'; import { EthnicityFilterCard } from './EthnicityFilterCard';
import { PoiDistanceFilterCard } from './PoiDistanceFilterCard'; import { PoiDistanceFilterCard } from './PoiDistanceFilterCard';
import { SchoolFilterCard } from './SchoolFilterCard'; import { SchoolFilterCard } from './SchoolFilterCard';
import { SpecificCrimeFilterCard } from './SpecificCrimeFilterCard'; import { VariantFilterCard } from './VariantFilterCard';
import { ElectionVoteShareFilterCard } from './ElectionVoteShareFilterCard'; import { ElectionVoteShareFilterCard } from './ElectionVoteShareFilterCard';
import { EnumFeatureFilterCard } from './EnumFeatureFilterCard'; import { EnumFeatureFilterCard } from './EnumFeatureFilterCard';
import { NumericFeatureFilterCard } from './NumericFeatureFilterCard'; import { NumericFeatureFilterCard } from './NumericFeatureFilterCard';
@ -40,6 +60,10 @@ interface ActiveFilterListProps {
destinationDropdownPortal: boolean; destinationDropdownPortal: boolean;
isGroupExpanded: (name: string) => boolean; isGroupExpanded: (name: string) => boolean;
onToggleGroup: (name: string) => void; onToggleGroup: (name: string) => void;
/** Registers a group wrapper so an expanded group can be scrolled into view. */
registerGroup?: (name: string) => (node: HTMLDivElement | null) => void;
/** Notifies the reveal-on-expand mechanism when a group is toggled. */
onGroupToggle?: (name: string, willExpand: boolean) => void;
onFilterChange: (name: string, value: [number, number] | string[]) => void; onFilterChange: (name: string, value: [number, number] | string[]) => void;
onRemoveFilter: (name: string) => void; onRemoveFilter: (name: string) => void;
onDragStart: (name: string, initialValue?: [number, number]) => void; onDragStart: (name: string, initialValue?: [number, number]) => void;
@ -76,6 +100,8 @@ export function ActiveFilterList({
destinationDropdownPortal, destinationDropdownPortal,
isGroupExpanded, isGroupExpanded,
onToggleGroup, onToggleGroup,
registerGroup,
onGroupToggle,
onFilterChange, onFilterChange,
onRemoveFilter, onRemoveFilter,
onDragStart, onDragStart,
@ -152,10 +178,11 @@ export function ActiveFilterList({
if (isSpecificCrimeFilterName(feature.name)) { if (isSpecificCrimeFilterName(feature.name)) {
const specificCrimeBackendName = getSpecificCrimeFeatureName(feature.name); const specificCrimeBackendName = getSpecificCrimeFeatureName(feature.name);
return ( return (
<SpecificCrimeFilterCard <VariantFilterCard
key={feature.name} key={feature.name}
config={SPECIFIC_CRIME_VARIANT_CONFIG}
features={features} features={features}
crimeFeature={feature} variantFeature={feature}
filters={filters} filters={filters}
activeFeature={activeFeature} activeFeature={activeFeature}
dragValue={dragValue} dragValue={dragValue}
@ -177,6 +204,91 @@ export function ActiveFilterList({
); );
} }
if (isCrimeSeverityFilterName(feature.name)) {
const crimeSeverityBackendName = getCrimeSeverityFeatureName(feature.name);
const crimeSeverityFilterName = getCrimeSeverityFilterName(feature.name);
if (!crimeSeverityFilterName) return null;
return (
<VariantFilterCard
key={feature.name}
config={getCrimeSeverityVariantConfig(crimeSeverityFilterName)}
features={features}
variantFeature={feature}
filters={filters}
activeFeature={activeFeature}
dragValue={dragValue}
pinnedFeature={pinnedFeature}
filterImpact={
crimeSeverityBackendName ? filterImpacts?.[crimeSeverityBackendName] : undefined
}
percentileScale={
crimeSeverityBackendName ? percentileScales.get(crimeSeverityBackendName) : undefined
}
onFilterChange={onFilterChange}
onDragStart={onDragStart}
onDragChange={onDragChange}
onDragEnd={onDragEnd}
onTogglePin={onTogglePin}
onShowInfo={onShowInfo}
onRemove={() => onRemoveFilter(feature.name)}
/>
);
}
if (isQualificationFilterName(feature.name)) {
const qualificationBackendName = getQualificationFeatureName(feature.name);
return (
<VariantFilterCard
key={feature.name}
config={QUALIFICATION_VARIANT_CONFIG}
features={features}
variantFeature={feature}
filters={filters}
activeFeature={activeFeature}
dragValue={dragValue}
pinnedFeature={pinnedFeature}
filterImpact={
qualificationBackendName ? filterImpacts?.[qualificationBackendName] : undefined
}
percentileScale={
qualificationBackendName ? percentileScales.get(qualificationBackendName) : undefined
}
onFilterChange={onFilterChange}
onDragStart={onDragStart}
onDragChange={onDragChange}
onDragEnd={onDragEnd}
onTogglePin={onTogglePin}
onShowInfo={onShowInfo}
onRemove={() => onRemoveFilter(feature.name)}
/>
);
}
if (isTenureFilterName(feature.name)) {
const tenureBackendName = getTenureFeatureName(feature.name);
return (
<VariantFilterCard
key={feature.name}
config={TENURE_VARIANT_CONFIG}
features={features}
variantFeature={feature}
filters={filters}
activeFeature={activeFeature}
dragValue={dragValue}
pinnedFeature={pinnedFeature}
filterImpact={tenureBackendName ? filterImpacts?.[tenureBackendName] : undefined}
percentileScale={tenureBackendName ? percentileScales.get(tenureBackendName) : undefined}
onFilterChange={onFilterChange}
onDragStart={onDragStart}
onDragChange={onDragChange}
onDragEnd={onDragEnd}
onTogglePin={onTogglePin}
onShowInfo={onShowInfo}
onRemove={() => onRemoveFilter(feature.name)}
/>
);
}
if (isElectionVoteShareFilterName(feature.name)) { if (isElectionVoteShareFilterName(feature.name)) {
const electionVoteShareBackendName = getElectionVoteShareFeatureName(feature.name); const electionVoteShareBackendName = getElectionVoteShareFeatureName(feature.name);
return ( return (
@ -298,11 +410,14 @@ export function ActiveFilterList({
if (count === 0) return null; if (count === 0) return null;
const expanded = isGroupExpanded(group.name); const expanded = isGroupExpanded(group.name);
return ( return (
<div key={group.name} className="shrink-0"> <div key={group.name} className="shrink-0" ref={registerGroup?.(group.name)}>
<CollapsibleGroupHeader <CollapsibleGroupHeader
name={group.name} name={group.name}
expanded={expanded} expanded={expanded}
onToggle={() => onToggleGroup(group.name)} onToggle={() => {
onToggleGroup(group.name);
onGroupToggle?.(group.name, !expanded);
}}
className="sticky top-0 z-30 px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 hover:bg-warm-200 dark:hover:bg-warm-800" className="sticky top-0 z-30 px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 hover:bg-warm-200 dark:hover:bg-warm-800"
> >
<span className="text-xs font-medium text-warm-400 dark:text-warm-500">{count}</span> <span className="text-xs font-medium text-warm-400 dark:text-warm-500">{count}</span>

View file

@ -1,6 +1,7 @@
import type { RefObject } from 'react'; import { useCallback, type MutableRefObject, type RefObject } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useRevealOnExpand } from '../../../hooks/useRevealOnExpand';
import type { AiFilterErrorType } from '../../../hooks/useAiFilters'; import type { AiFilterErrorType } from '../../../hooks/useAiFilters';
import type { TravelTimeEntry } from '../../../hooks/useTravelTime'; import type { TravelTimeEntry } from '../../../hooks/useTravelTime';
import type { PercentileScale } from '../../../lib/format'; import type { PercentileScale } from '../../../lib/format';
@ -107,6 +108,16 @@ export function ActiveFiltersPanel({
onTravelTimeToggleNoBuses, onTravelTimeToggleNoBuses,
}: ActiveFiltersPanelProps) { }: ActiveFiltersPanelProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
// Compose the parent-owned scrollRef (used to locate newly added filters) with
// the reveal-on-expand container ref so both point at the same scroll element.
const setScrollNode = useCallback(
(node: HTMLDivElement | null) => {
(scrollRef as MutableRefObject<HTMLDivElement | null>).current = node;
setContainer(node);
},
[scrollRef, setContainer]
);
return ( return (
<div <div
@ -156,7 +167,10 @@ export function ActiveFiltersPanel({
</button> </button>
{!collapsed && ( {!collapsed && (
<div ref={scrollRef} className="md:min-h-0 md:flex-1 md:overflow-y-auto overflow-x-hidden"> <div
ref={setScrollNode}
className="md:min-h-0 md:flex-1 md:overflow-y-auto overflow-x-hidden"
>
<AiFilterInput <AiFilterInput
loading={aiFilterLoading} loading={aiFilterLoading}
error={aiFilterError} error={aiFilterError}
@ -196,6 +210,8 @@ export function ActiveFiltersPanel({
destinationDropdownPortal={destinationDropdownPortal} destinationDropdownPortal={destinationDropdownPortal}
isGroupExpanded={isGroupExpanded} isGroupExpanded={isGroupExpanded}
onToggleGroup={onToggleGroup} onToggleGroup={onToggleGroup}
registerGroup={registerGroup}
onGroupToggle={revealGroupOnToggle}
onFilterChange={onFilterChange} onFilterChange={onFilterChange}
onRemoveFilter={onRemoveFilter} onRemoveFilter={onRemoveFilter}
onDragStart={onDragStart} onDragStart={onDragStart}

View file

@ -1,15 +1,27 @@
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useRevealOnExpand } from '../../../hooks/useRevealOnExpand';
import type { TravelTimeEntry, TransportMode } from '../../../hooks/useTravelTime'; import type { TravelTimeEntry, TransportMode } from '../../../hooks/useTravelTime';
import type { FeatureMeta } from '../../../types'; import type { FeatureMeta } from '../../../types';
import { ChevronIcon } from '../../ui/icons'; import { ChevronIcon } from '../../ui/icons';
import FeatureBrowser from '../FeatureBrowser'; import FeatureBrowser from '../FeatureBrowser';
import { SPECIFIC_CRIMES_FILTER_NAME, isSpecificCrimeFilterName } from '../../../lib/crime-filter'; import { SPECIFIC_CRIMES_FILTER_NAME, isSpecificCrimeFilterName } from '../../../lib/crime-filter';
import {
CRIME_SEVERITY_FILTER_NAMES,
getCrimeSeverityFilterName,
isCrimeSeverityFilterName,
type CrimeSeverityFilterName,
} from '../../../lib/crime-severity-filter';
import { import {
ELECTION_VOTE_SHARE_FILTER_NAME, ELECTION_VOTE_SHARE_FILTER_NAME,
isElectionVoteShareFilterName, isElectionVoteShareFilterName,
} from '../../../lib/election-filter'; } from '../../../lib/election-filter';
import { ETHNICITIES_FILTER_NAME, isEthnicityFilterName } from '../../../lib/ethnicity-filter'; import { ETHNICITIES_FILTER_NAME, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
import {
QUALIFICATIONS_FILTER_NAME,
isQualificationFilterName,
} from '../../../lib/qualification-filter';
import { TENURE_FILTER_NAME, isTenureFilterName } from '../../../lib/tenure-filter';
import { SCHOOL_FILTER_NAME, isSchoolFilterName } from '../../../lib/school-filter'; import { SCHOOL_FILTER_NAME, isSchoolFilterName } from '../../../lib/school-filter';
import { import {
POI_DISTANCE_FILTER_NAME, POI_DISTANCE_FILTER_NAME,
@ -27,8 +39,11 @@ interface AddFilterPanelProps {
pinnedFeature: string | null; pinnedFeature: string | null;
defaultSchoolFeatureName: string | null; defaultSchoolFeatureName: string | null;
defaultSpecificCrimeFeatureName: string | null; defaultSpecificCrimeFeatureName: string | null;
defaultCrimeSeverityFeatureNames: Record<CrimeSeverityFilterName, string | null>;
defaultElectionVoteShareFeatureName: string | null; defaultElectionVoteShareFeatureName: string | null;
defaultEthnicityFeatureName: string | null; defaultEthnicityFeatureName: string | null;
defaultQualificationFeatureName: string | null;
defaultTenureFeatureName: string | null;
defaultPoiFilterFeatureNames: Record<PoiFilterName, string | null>; defaultPoiFilterFeatureNames: Record<PoiFilterName, string | null>;
openInfoFeature?: string | null; openInfoFeature?: string | null;
travelTimeEntries: TravelTimeEntry[]; travelTimeEntries: TravelTimeEntry[];
@ -49,8 +64,11 @@ export function AddFilterPanel({
pinnedFeature, pinnedFeature,
defaultSchoolFeatureName, defaultSchoolFeatureName,
defaultSpecificCrimeFeatureName, defaultSpecificCrimeFeatureName,
defaultCrimeSeverityFeatureNames,
defaultElectionVoteShareFeatureName, defaultElectionVoteShareFeatureName,
defaultEthnicityFeatureName, defaultEthnicityFeatureName,
defaultQualificationFeatureName,
defaultTenureFeatureName,
defaultPoiFilterFeatureNames, defaultPoiFilterFeatureNames,
openInfoFeature, openInfoFeature,
travelTimeEntries, travelTimeEntries,
@ -63,6 +81,7 @@ export function AddFilterPanel({
onUpgradeClick, onUpgradeClick,
}: AddFilterPanelProps) { }: AddFilterPanelProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
const browserPinnedFeature = const browserPinnedFeature =
pinnedFeature && isSchoolFilterName(pinnedFeature) pinnedFeature && isSchoolFilterName(pinnedFeature)
@ -73,9 +92,15 @@ export function AddFilterPanel({
? ELECTION_VOTE_SHARE_FILTER_NAME ? ELECTION_VOTE_SHARE_FILTER_NAME
: pinnedFeature && isEthnicityFilterName(pinnedFeature) : pinnedFeature && isEthnicityFilterName(pinnedFeature)
? ETHNICITIES_FILTER_NAME ? ETHNICITIES_FILTER_NAME
: pinnedFeature && isPoiDistanceFilterName(pinnedFeature) : pinnedFeature && isQualificationFilterName(pinnedFeature)
? (getPoiFilterName(pinnedFeature) ?? POI_DISTANCE_FILTER_NAME) ? QUALIFICATIONS_FILTER_NAME
: pinnedFeature; : pinnedFeature && isTenureFilterName(pinnedFeature)
? TENURE_FILTER_NAME
: pinnedFeature && isPoiDistanceFilterName(pinnedFeature)
? (getPoiFilterName(pinnedFeature) ?? POI_DISTANCE_FILTER_NAME)
: pinnedFeature && isCrimeSeverityFilterName(pinnedFeature)
? (getCrimeSeverityFilterName(pinnedFeature) ?? pinnedFeature)
: pinnedFeature;
const handleTogglePin = (name: string) => { const handleTogglePin = (name: string) => {
if (name === SCHOOL_FILTER_NAME) { if (name === SCHOOL_FILTER_NAME) {
@ -94,6 +119,20 @@ export function AddFilterPanel({
if (defaultEthnicityFeatureName) onTogglePin(defaultEthnicityFeatureName); if (defaultEthnicityFeatureName) onTogglePin(defaultEthnicityFeatureName);
return; return;
} }
if (name === QUALIFICATIONS_FILTER_NAME) {
if (defaultQualificationFeatureName) onTogglePin(defaultQualificationFeatureName);
return;
}
if (name === TENURE_FILTER_NAME) {
if (defaultTenureFeatureName) onTogglePin(defaultTenureFeatureName);
return;
}
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
const defaultSeverityFeatureName =
defaultCrimeSeverityFeatureNames[name as CrimeSeverityFilterName];
if (defaultSeverityFeatureName) onTogglePin(defaultSeverityFeatureName);
return;
}
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) { if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
const defaultPoiFeatureName = defaultPoiFilterFeatureNames[name as PoiFilterName]; const defaultPoiFeatureName = defaultPoiFilterFeatureNames[name as PoiFilterName];
if (defaultPoiFeatureName) onTogglePin(defaultPoiFeatureName); if (defaultPoiFeatureName) onTogglePin(defaultPoiFeatureName);
@ -123,7 +162,7 @@ export function AddFilterPanel({
{(!collapsed || !isLicensed) && ( {(!collapsed || !isLicensed) && (
<div className="flex min-h-0 flex-1 flex-col"> <div className="flex min-h-0 flex-1 flex-col">
{!collapsed && ( {!collapsed && (
<div className="min-h-0 flex-1 overflow-y-auto"> <div ref={setContainer} className="min-h-0 flex-1 overflow-y-auto">
<FeatureBrowser <FeatureBrowser
availableFeatures={availableFeatures} availableFeatures={availableFeatures}
allFeatures={allFeatures} allFeatures={allFeatures}
@ -135,6 +174,8 @@ export function AddFilterPanel({
onClearOpenInfoFeature={onClearOpenInfoFeature} onClearOpenInfoFeature={onClearOpenInfoFeature}
travelTimeEntries={travelTimeEntries} travelTimeEntries={travelTimeEntries}
onAddTravelTimeEntry={onAddTravelTimeEntry} onAddTravelTimeEntry={onAddTravelTimeEntry}
registerGroup={registerGroup}
onGroupToggle={revealGroupOnToggle}
/> />
</div> </div>
)} )}

View file

@ -5,23 +5,22 @@ import { ChevronIcon } from '../../ui/icons';
import { FeatureActions } from '../../ui/FeatureIcons'; import { FeatureActions } from '../../ui/FeatureIcons';
import { FeatureLabel } from '../../ui/FeatureLabel'; import { FeatureLabel } from '../../ui/FeatureLabel';
import type { FeatureFilters, FeatureMeta } from '../../../types'; import type { FeatureFilters, FeatureMeta } from '../../../types';
import type { VariantFilterConfig } from '../../../lib/variant-filter';
import { formatNumber, type PercentileScale } from '../../../lib/format'; import { formatNumber, type PercentileScale } from '../../../lib/format';
import { getFeatureIcon } from '../../../lib/feature-icons'; import { getFeatureIcon } from '../../../lib/feature-icons';
import { getGroupIcon } from '../../../lib/group-icons'; import { getGroupIcon } from '../../../lib/group-icons';
import {
SPECIFIC_CRIMES_FILTER_NAME,
SPECIFIC_CRIME_FEATURE_NAMES,
clampSpecificCrimeRange,
getDefaultSpecificCrimeFeatureName,
getSpecificCrimeFeatureName,
getSpecificCrimeFilterMeta,
replaceSpecificCrimeFilterKeySelection,
} from '../../../lib/crime-filter';
import { SliderLabels } from './SliderLabels'; import { SliderLabels } from './SliderLabels';
export function SpecificCrimeFilterCard({ /**
* One card for a "pick a variant, filter its range" aggregate filter
* (specific crimes, qualifications, ). Behaviour is variant-agnostic: the
* `config` supplies the dropdown options, labels and helper functions so the
* same component backs every such filter (see [`VariantFilterConfig`]).
*/
export function VariantFilterCard({
config,
features, features,
crimeFeature, variantFeature,
filters, filters,
activeFeature, activeFeature,
dragValue, dragValue,
@ -36,8 +35,9 @@ export function SpecificCrimeFilterCard({
onShowInfo, onShowInfo,
onRemove, onRemove,
}: { }: {
config: VariantFilterConfig;
features: FeatureMeta[]; features: FeatureMeta[];
crimeFeature: FeatureMeta; variantFeature: FeatureMeta;
filters: FeatureFilters; filters: FeatureFilters;
activeFeature: string | null; activeFeature: string | null;
dragValue: [number, number] | null; dragValue: [number, number] | null;
@ -53,27 +53,62 @@ export function SpecificCrimeFilterCard({
onRemove: () => void; onRemove: () => void;
}) { }) {
const { t } = useTranslation(); const { t } = useTranslation();
const specificCrimeMeta = getSpecificCrimeFilterMeta(features); const variantMeta = config.getFilterMeta(features);
const crimeOptions = SPECIFIC_CRIME_FEATURE_NAMES.map((name) =>
features.find((feature) => feature.name === name)
).filter((feature): feature is FeatureMeta => Boolean(feature));
const selectedFeatureName = const selectedFeatureName =
getSpecificCrimeFeatureName(crimeFeature.name) ?? getDefaultSpecificCrimeFeatureName(features); config.getFeatureName(variantFeature.name) ?? config.getDefaultFeatureName(features);
const selectedFeature = selectedFeatureName const selectedFeature = selectedFeatureName
? features.find((feature) => feature.name === selectedFeatureName) ? features.find((feature) => feature.name === selectedFeatureName)
: undefined; : undefined;
if (!selectedFeature || crimeOptions.length === 0 || !selectedFeatureName) return null; // Optional secondary axis: the same variant measured over a different window
// (e.g. 7- vs 2-year crime rates). The current window comes from the selected
// feature so the dropdown can re-point each option into that window.
const windowConfig = config.window;
const currentWindow =
(windowConfig && selectedFeatureName ? windowConfig.getWindow(selectedFeatureName) : null) ??
windowConfig?.options[0]?.id ??
null;
const isActive = activeFeature === crimeFeature.name; const variantOptions = config.featureNames
const isPinned = pinnedFeature === crimeFeature.name; .map((canonicalName) => {
const optionName =
windowConfig && currentWindow
? windowConfig.withWindow(canonicalName, currentWindow)
: canonicalName;
const feature = features.find((f) => f.name === optionName);
if (!feature) return null;
return {
value: optionName,
label: ts(config.getOptionLabelSource ? config.getOptionLabelSource(optionName) : optionName),
feature,
};
})
.filter((option): option is { value: string; label: string; feature: FeatureMeta } =>
Boolean(option)
);
// Only offer windows that actually exist for the selected variant, so a
// missing backend column can't strand the card on an empty selection.
const windowOptions =
windowConfig && selectedFeatureName
? windowConfig.options.filter((option) =>
features.some(
(f) => f.name === windowConfig.withWindow(selectedFeatureName, option.id)
)
)
: [];
if (!selectedFeature || variantOptions.length === 0 || !selectedFeatureName) return null;
const isActive = activeFeature === variantFeature.name;
const isPinned = pinnedFeature === variantFeature.name;
const hist = selectedFeature.histogram; const hist = selectedFeature.histogram;
const dataMin = hist?.min ?? selectedFeature.min ?? 0; const dataMin = hist?.min ?? selectedFeature.min ?? 0;
const dataMax = hist?.max ?? selectedFeature.max ?? 100; const dataMax = hist?.max ?? selectedFeature.max ?? 100;
const displayValue = const displayValue =
isActive && dragValue isActive && dragValue
? dragValue ? dragValue
: (filters[crimeFeature.name] as [number, number]) || [dataMin, dataMax]; : (filters[variantFeature.name] as [number, number]) || [dataMin, dataMax];
const scale = percentileScale; const scale = percentileScale;
const clampMin = displayValue[0] <= dataMin; const clampMin = displayValue[0] <= dataMin;
const clampMax = displayValue[1] >= dataMax; const clampMax = displayValue[1] >= dataMax;
@ -89,14 +124,14 @@ export function SpecificCrimeFilterCard({
clampMax ? (selectedFeature.max ?? dataMax) : displayValue[1], clampMax ? (selectedFeature.max ?? dataMax) : displayValue[1],
]; ];
const replaceCrimeFeature = (nextFeatureName: string) => { const replaceVariantFeature = (nextFeatureName: string) => {
const nextName = replaceSpecificCrimeFilterKeySelection(crimeFeature.name, nextFeatureName); const nextName = config.replaceFilterKeySelection(variantFeature.name, nextFeatureName);
if (nextName === crimeFeature.name) return; if (nextName === variantFeature.name) return;
const nextFeature = features.find((feature) => feature.name === nextFeatureName); const nextFeature = features.find((feature) => feature.name === nextFeatureName);
const nextDataMin = nextFeature?.histogram?.min ?? nextFeature?.min ?? 0; const nextDataMin = nextFeature?.histogram?.min ?? nextFeature?.min ?? 0;
const nextDataMax = nextFeature?.histogram?.max ?? nextFeature?.max ?? Math.max(1, dataMax); const nextDataMax = nextFeature?.histogram?.max ?? nextFeature?.max ?? Math.max(1, dataMax);
const nextRange = clampSpecificCrimeRange( const nextRange = config.clampRange(
[ [
displayValue[0] <= dataMin ? nextDataMin : displayValue[0], displayValue[0] <= dataMin ? nextDataMin : displayValue[0],
displayValue[1] >= dataMax ? nextDataMax : displayValue[1], displayValue[1] >= dataMax ? nextDataMax : displayValue[1],
@ -108,6 +143,11 @@ export function SpecificCrimeFilterCard({
if (isPinned) onTogglePin(nextName); if (isPinned) onTogglePin(nextName);
}; };
const switchWindow = (windowId: string) => {
if (!windowConfig || windowId === currentWindow) return;
replaceVariantFeature(windowConfig.withWindow(selectedFeatureName, windowId));
};
const mobileIconClass = 'w-4 h-4 text-teal-600 dark:text-teal-400 shrink-0'; const mobileIconClass = 'w-4 h-4 text-teal-600 dark:text-teal-400 shrink-0';
const mobileIcon = const mobileIcon =
getFeatureIcon(selectedFeature.name, mobileIconClass) || getFeatureIcon(selectedFeature.name, mobileIconClass) ||
@ -118,7 +158,7 @@ export function SpecificCrimeFilterCard({
return ( return (
<div <div
data-filter-name={SPECIFIC_CRIMES_FILTER_NAME} data-filter-name={config.filterName}
className={`space-y-1.5 px-2 py-1.5 rounded ${ className={`space-y-1.5 px-2 py-1.5 rounded ${
isActive isActive
? 'ring-2 ring-teal-400 bg-teal-50 dark:bg-teal-900/30' ? 'ring-2 ring-teal-400 bg-teal-50 dark:bg-teal-900/30'
@ -129,7 +169,7 @@ export function SpecificCrimeFilterCard({
> >
<div className="relative z-10 flex items-center justify-between gap-1"> <div className="relative z-10 flex items-center justify-between gap-1">
<FeatureLabel <FeatureLabel
feature={specificCrimeMeta} feature={variantMeta}
size="sm" size="sm"
className="min-w-0 shrink" className="min-w-0 shrink"
hideIconOnMobile hideIconOnMobile
@ -137,7 +177,7 @@ export function SpecificCrimeFilterCard({
/> />
<FeatureActions <FeatureActions
feature={selectedFeature} feature={selectedFeature}
actionName={crimeFeature.name} actionName={variantFeature.name}
isPinned={isPinned} isPinned={isPinned}
isPreviewing={isActive} isPreviewing={isActive}
onTogglePin={onTogglePin} onTogglePin={onTogglePin}
@ -146,28 +186,65 @@ export function SpecificCrimeFilterCard({
/> />
</div> </div>
<div> {/* A single-variant filter (e.g. Serious/Minor crime) has nothing to pick,
<label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500"> so the dropdown is hidden and only the window toggle + slider remain. */}
{t('filters.crimeType')} {variantOptions.length > 1 && (
</label> <div>
<div className="relative"> <label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500">
<select {t(config.dropdownLabelKey)}
value={selectedFeatureName} </label>
onChange={(e) => replaceCrimeFeature(e.target.value)} <div className="relative">
className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50" <select
> value={selectedFeatureName}
{crimeOptions.map((option) => ( onChange={(e) => replaceVariantFeature(e.target.value)}
<option key={option.name} value={option.name}> className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50"
{ts(option.name)} >
</option> {variantOptions.map((option) => (
))} <option key={option.value} value={option.value}>
</select> {option.label}
<ChevronIcon </option>
direction="down" ))}
className="pointer-events-none absolute right-2 top-1/2 h-4 w-4 -translate-y-1/2 text-warm-400 dark:text-warm-500" </select>
/> <ChevronIcon
direction="down"
className="pointer-events-none absolute right-2 top-1/2 h-4 w-4 -translate-y-1/2 text-warm-400 dark:text-warm-500"
/>
</div>
</div> </div>
</div> )}
{windowConfig && currentWindow && windowOptions.length > 1 && (
<div>
{windowConfig.labelKey && (
<label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500">
{t(windowConfig.labelKey)}
</label>
)}
<div
role="group"
className="inline-flex rounded-md border border-warm-200 bg-warm-50 p-0.5 dark:border-warm-700 dark:bg-navy-900"
>
{windowOptions.map((option) => {
const active = option.id === currentWindow;
return (
<button
key={option.id}
type="button"
aria-pressed={active}
onClick={() => switchWindow(option.id)}
className={`rounded px-2.5 py-1 text-xs font-medium transition-colors ${
active
? 'bg-teal-600 text-white shadow-sm'
: 'text-warm-500 hover:text-navy-950 dark:text-warm-400 dark:hover:text-warm-100'
}`}
>
{t(option.labelKey)}
</button>
);
})}
</div>
</div>
)}
<div className="flex items-start gap-1.5 md:block"> <div className="flex items-start gap-1.5 md:block">
{mobileIcon && <div className="shrink-0 pt-0.5 md:hidden">{mobileIcon}</div>} {mobileIcon && <div className="shrink-0 pt-0.5 md:hidden">{mobileIcon}</div>}
@ -198,7 +275,7 @@ export function SpecificCrimeFilterCard({
max >= (selectedFeature.max ?? dataMax) ? dataMax : max, max >= (selectedFeature.max ?? dataMax) ? dataMax : max,
]) ])
} }
onPointerDown={() => onDragStart(crimeFeature.name, displayValue)} onPointerDown={() => onDragStart(variantFeature.name, displayValue)}
onPointerUp={() => onDragEnd()} onPointerUp={() => onDragEnd()}
/> />
<SliderLabels <SliderLabels
@ -211,7 +288,7 @@ export function SpecificCrimeFilterCard({
raw={selectedFeature.raw} raw={selectedFeature.raw}
feature={selectedFeature} feature={selectedFeature}
onValueChange={(v) => onValueChange={(v) =>
onFilterChange(crimeFeature.name, clampSpecificCrimeRange(v, selectedFeature)) onFilterChange(variantFeature.name, config.clampRange(v, selectedFeature))
} }
/> />
{filterImpact != null && filterImpact > 0 && ( {filterImpact != null && filterImpact > 0 && (

View file

@ -236,8 +236,8 @@ export function DesktopMapPage({
onClick={onToggleActualListings} onClick={onToggleActualListings}
aria-pressed={actualListingsEnabled} aria-pressed={actualListingsEnabled}
aria-busy={actualListingsLoading} aria-busy={actualListingsLoading}
aria-label={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'} aria-label={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
title={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'} title={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`} className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
> >
{actualListingsLoading ? ( {actualListingsLoading ? (
@ -246,16 +246,17 @@ export function DesktopMapPage({
<HouseIcon className="h-5 w-5" /> <HouseIcon className="h-5 w-5" />
)} )}
<span className="text-sm font-medium"> <span className="text-sm font-medium">
Listings{actualListingsEnabled ? ` (${actualListings.length})` : ''} {t('map.actualListings.label')}{actualListingsEnabled ? ` (${actualListings.length})` : ''}
</span> </span>
</button> </button>
)} )}
<button <button
data-tutorial="overlays-button"
onClick={onToggleOverlayPane} onClick={onToggleOverlayPane}
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`} className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
> >
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} /> <EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
<span className="text-sm font-medium">Overlays</span> <span className="text-sm font-medium">{t('overlays.heading')}</span>
</button> </button>
<button <button
data-tutorial="poi-button" data-tutorial="poi-button"

View file

@ -1,4 +1,5 @@
import { Suspense, type MutableRefObject, type ReactNode } from 'react'; import { Suspense, type MutableRefObject, type ReactNode } from 'react';
import { useTranslation } from 'react-i18next';
import type { import type {
ActualListing, ActualListing,
@ -133,6 +134,7 @@ export function MobileMapPage({
upgradeModal, upgradeModal,
editingBar, editingBar,
}: MobileMapPageProps) { }: MobileMapPageProps) {
const { t } = useTranslation();
const floatingPaneAvailableHeight = `max(12rem, calc(100dvh - ${Math.ceil( const floatingPaneAvailableHeight = `max(12rem, calc(100dvh - ${Math.ceil(
bottomScreenInset bottomScreenInset
)}px - 7rem))`; )}px - 7rem))`;
@ -204,8 +206,16 @@ export function MobileMapPage({
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`} className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
aria-pressed={actualListingsEnabled} aria-pressed={actualListingsEnabled}
aria-busy={actualListingsLoading} aria-busy={actualListingsLoading}
aria-label={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'} aria-label={
title={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'} actualListingsEnabled
? t('map.actualListings.hide')
: t('map.actualListings.show')
}
title={
actualListingsEnabled
? t('map.actualListings.hide')
: t('map.actualListings.show')
}
> >
{actualListingsLoading ? ( {actualListingsLoading ? (
<SpinnerIcon className="h-5 w-5 animate-spin" /> <SpinnerIcon className="h-5 w-5 animate-spin" />
@ -217,7 +227,7 @@ export function MobileMapPage({
<button <button
onClick={onToggleOverlayPane} onClick={onToggleOverlayPane}
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`} className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
aria-label="Overlays" aria-label={t('overlays.heading')}
> >
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} /> <EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
</button> </button>

View file

@ -6,8 +6,11 @@ import type { HexagonLocation } from '../../../lib/external-search';
import type { useMapData } from '../../../hooks/useMapData'; import type { useMapData } from '../../../hooks/useMapData';
import { resolveTransitVariant, type TravelTimeEntry } from '../../../hooks/useTravelTime'; import { resolveTransitVariant, type TravelTimeEntry } from '../../../hooks/useTravelTime';
import { getSpecificCrimeFeatureName } from '../../../lib/crime-filter'; import { getSpecificCrimeFeatureName } from '../../../lib/crime-filter';
import { getCrimeSeverityFeatureName } from '../../../lib/crime-severity-filter';
import { getElectionVoteShareFeatureName } from '../../../lib/election-filter'; import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter'; import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
import { getQualificationFeatureName } from '../../../lib/qualification-filter';
import { getTenureFeatureName } from '../../../lib/tenure-filter';
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter'; import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
import { getSchoolBackendFeatureName } from '../../../lib/school-filter'; import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
@ -23,8 +26,11 @@ export function getMapPageBackendFeatureName(featureName: string): string {
return ( return (
getSchoolBackendFeatureName(featureName) ?? getSchoolBackendFeatureName(featureName) ??
getSpecificCrimeFeatureName(featureName) ?? getSpecificCrimeFeatureName(featureName) ??
getCrimeSeverityFeatureName(featureName) ??
getElectionVoteShareFeatureName(featureName) ?? getElectionVoteShareFeatureName(featureName) ??
getEthnicityFeatureName(featureName) ?? getEthnicityFeatureName(featureName) ??
getQualificationFeatureName(featureName) ??
getTenureFeatureName(featureName) ??
getPoiDistanceFeatureName(featureName) ?? getPoiDistanceFeatureName(featureName) ??
featureName featureName
); );

View file

@ -1,16 +1,20 @@
import { useTranslation } from 'react-i18next';
interface IndeterminateProgressBarProps { interface IndeterminateProgressBarProps {
show: boolean; show: boolean;
className?: string; className?: string;
} }
export function IndeterminateProgressBar({ show, className = '' }: IndeterminateProgressBarProps) { export function IndeterminateProgressBar({ show, className = '' }: IndeterminateProgressBarProps) {
const { t } = useTranslation();
if (!show) return null; if (!show) return null;
return ( return (
<div <div
role="progressbar" role="progressbar"
aria-busy="true" aria-busy="true"
aria-valuetext="loading" aria-valuetext={t('common.loading')}
className={`pointer-events-none absolute top-0 left-0 right-0 z-30 h-0.5 overflow-hidden bg-teal-500/10 dark:bg-teal-400/10 animate-fade-in ${className}`} className={`pointer-events-none absolute top-0 left-0 right-0 z-30 h-0.5 overflow-hidden bg-teal-500/10 dark:bg-teal-400/10 animate-fade-in ${className}`}
> >
<div className="h-full w-1/4 bg-teal-500 dark:bg-teal-400 animate-indeterminate-progress" /> <div className="h-full w-1/4 bg-teal-500 dark:bg-teal-400 animate-indeterminate-progress" />

View file

@ -1,4 +1,5 @@
import { useState, useCallback, useRef } from 'react'; import { useState, useCallback, useRef } from 'react';
import { useTranslation } from 'react-i18next';
import type { FeatureFilters } from '../types'; import type { FeatureFilters } from '../types';
import { apiUrl, authHeaders, logNonAbortError } from '../lib/api'; import { apiUrl, authHeaders, logNonAbortError } from '../lib/api';
@ -88,6 +89,7 @@ function buildSummary(
} }
export function useAiFilters(): UseAiFiltersResult { export function useAiFilters(): UseAiFiltersResult {
const { t } = useTranslation();
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [errorType, setErrorType] = useState<AiFilterErrorType | null>(null); const [errorType, setErrorType] = useState<AiFilterErrorType | null>(null);
@ -170,14 +172,14 @@ export function useAiFilters(): UseAiFiltersResult {
} catch (err) { } catch (err) {
if (controller.signal.aborted) return null; if (controller.signal.aborted) return null;
logNonAbortError('ai-filters', err); logNonAbortError('ai-filters', err);
const message = err instanceof Error ? err.message : 'Failed to generate filters'; const message = err instanceof Error ? err.message : t('aiFilter.generateFailed');
setErrorType('error'); setErrorType('error');
setError(message); setError(message);
setLoading(false); setLoading(false);
return null; return null;
} }
}, },
[] [t]
); );
return { fetchAiFilters, loading, error, errorType, notes, summary }; return { fetchAiFilters, loading, error, errorType, notes, summary };

View file

@ -1,4 +1,5 @@
import { useState, useEffect, useCallback } from 'react'; import { useState, useEffect, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import pb from '../lib/pocketbase'; import pb from '../lib/pocketbase';
import { trackEvent } from '../lib/analytics'; import { trackEvent } from '../lib/analytics';
@ -31,6 +32,7 @@ function authRefreshInvalidated(error: unknown): boolean {
} }
export function useAuth() { export function useAuth() {
const { t } = useTranslation();
const [user, setUser] = useState<AuthUser | null>(() => { const [user, setUser] = useState<AuthUser | null>(() => {
if (pb.authStore.isValid && pb.authStore.record) { if (pb.authStore.isValid && pb.authStore.record) {
return recordToUser(pb.authStore.record); return recordToUser(pb.authStore.record);
@ -60,13 +62,13 @@ export function useAuth() {
setUser(recordToUser(result.record)); setUser(recordToUser(result.record));
trackEvent('Login', { method: 'email' }); trackEvent('Login', { method: 'email' });
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Login failed'; const msg = err instanceof Error ? err.message : t('auth.loginFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, []); }, [t]);
const register = useCallback(async (email: string, password: string) => { const register = useCallback(async (email: string, password: string) => {
setLoading(true); setLoading(true);
@ -83,13 +85,13 @@ export function useAuth() {
setUser(recordToUser(result.record)); setUser(recordToUser(result.record));
trackEvent('Register'); trackEvent('Register');
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Registration failed'; const msg = err instanceof Error ? err.message : t('auth.registrationFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, []); }, [t]);
const loginWithOAuth = useCallback(async (provider: string) => { const loginWithOAuth = useCallback(async (provider: string) => {
setLoading(true); setLoading(true);
@ -102,13 +104,13 @@ export function useAuth() {
setUser(recordToUser(result.record)); setUser(recordToUser(result.record));
trackEvent('Login', { method: provider }); trackEvent('Login', { method: provider });
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'OAuth login failed'; const msg = err instanceof Error ? err.message : t('auth.oauthFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, []); }, [t]);
const logout = useCallback(() => { const logout = useCallback(() => {
trackEvent('Logout'); trackEvent('Logout');
@ -122,13 +124,13 @@ export function useAuth() {
try { try {
await pb.collection('users').requestPasswordReset(email); await pb.collection('users').requestPasswordReset(email);
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Password reset request failed'; const msg = err instanceof Error ? err.message : t('auth.passwordResetFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, []); }, [t]);
const refreshAuth = useCallback(async () => { const refreshAuth = useCallback(async () => {
setLoading(true); setLoading(true);

View file

@ -0,0 +1,98 @@
import { act, renderHook, waitFor } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
create: vi.fn(),
getFullList: vi.fn(),
authStore: {
isValid: true,
record: { id: 'user-1' } as { id: string } | null,
onChange: () => () => {},
},
}));
vi.mock('../lib/pocketbase', () => ({
default: {
authStore: mocks.authStore,
collection: () => ({ getFullList: mocks.getFullList, create: mocks.create }),
},
}));
import { useClickedListings } from './useClickedListings';
describe('useClickedListings', () => {
beforeEach(() => {
mocks.create.mockReset().mockResolvedValue({});
mocks.getFullList.mockReset().mockResolvedValue([]);
mocks.authStore.isValid = true;
mocks.authStore.record = { id: 'user-1' };
});
afterEach(() => {
vi.clearAllMocks();
});
it('hydrates the visited set from PocketBase for the signed-in user', async () => {
mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
const { result } = renderHook(() => useClickedListings());
await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
});
it('records a click optimistically and persists it', async () => {
const { result } = renderHook(() => useClickedListings());
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
act(() => result.current.markClicked('https://example.com/b'));
// Visible immediately — no need to await the network round-trip.
expect(result.current.clickedUrls.has('https://example.com/b')).toBe(true);
expect(mocks.create).toHaveBeenCalledWith({
user: 'user-1',
url: 'https://example.com/b',
});
});
it('produces a new Set instance for a new url so deck.gl re-colours', async () => {
const { result } = renderHook(() => useClickedListings());
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
const before = result.current.clickedUrls;
act(() => result.current.markClicked('https://example.com/c'));
expect(result.current.clickedUrls).not.toBe(before);
});
it('does not re-persist an already visited url', async () => {
mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
const { result } = renderHook(() => useClickedListings());
await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
const before = result.current.clickedUrls;
act(() => result.current.markClicked('https://example.com/a'));
expect(mocks.create).not.toHaveBeenCalled();
expect(result.current.clickedUrls).toBe(before); // identity stable → no needless recompute
});
it('ignores empty urls', async () => {
const { result } = renderHook(() => useClickedListings());
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
act(() => result.current.markClicked(''));
act(() => result.current.markClicked(undefined));
expect(mocks.create).not.toHaveBeenCalled();
expect(result.current.clickedUrls.size).toBe(0);
});
it('recolours for anonymous users in-memory but does not persist', async () => {
mocks.authStore.isValid = false;
mocks.authStore.record = null;
const { result } = renderHook(() => useClickedListings());
act(() => result.current.markClicked('https://example.com/d'));
expect(result.current.clickedUrls.has('https://example.com/d')).toBe(true);
expect(mocks.create).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,82 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import pb from '../lib/pocketbase';
/** Per-user record of opened listings, stored in PocketBase so visited listings can be drawn
* in a distinct colour on the map and persist across devices. One row per (user, url). */
const CLICKED_LISTINGS_COLLECTION = 'clicked_listings';
function currentUserId(): string | null {
return pb.authStore.isValid && pb.authStore.record ? pb.authStore.record.id : null;
}
/** Tracks which listings the signed-in user has opened.
*
* Reads the user from the shared PocketBase authStore so it stays self-contained no need to
* thread a user id down through the map layer tree. Returns a Set for O(1) membership checks in
* the map's colour accessor; a fresh Set instance is produced on every change so it can double
* as a deck.gl `updateTrigger` (identity-compared).
*
* Writes are optimistic: `markClicked` updates local state synchronously so the pin recolours
* the instant it is clicked, independent of the PocketBase round-trip or the next listings fetch.
*/
export function useClickedListings() {
const [clickedUrls, setClickedUrls] = useState<Set<string>>(() => new Set());
const [userId, setUserId] = useState<string | null>(currentUserId);
const userIdRef = useRef(userId);
userIdRef.current = userId;
// Mirrors the latest committed set so markClicked can decide (synchronously) whether a url is
// new — without depending on the async state updater having run yet.
const clickedUrlsRef = useRef(clickedUrls);
clickedUrlsRef.current = clickedUrls;
// Follow login/logout via the shared authStore (also covers cross-tab auth changes).
useEffect(() => {
const unsubscribe = pb.authStore.onChange(() => setUserId(currentUserId()));
return unsubscribe;
}, []);
// Load the user's previously opened listings whenever the signed-in user changes. Signed-out
// users keep an in-memory set for the current session only (nothing to persist to).
useEffect(() => {
if (!userId) {
setClickedUrls(new Set());
return;
}
let cancelled = false;
pb.collection(CLICKED_LISTINGS_COLLECTION)
.getFullList({ filter: `user = "${userId}"`, fields: 'url' })
.then((records) => {
if (cancelled) return;
const urls = records
.map((record) => (record as { url?: unknown }).url)
.filter((url): url is string => typeof url === 'string');
setClickedUrls(new Set(urls));
})
.catch(() => {
// Visited history is a convenience; a failed load just starts the session empty.
});
return () => {
cancelled = true;
};
}, [userId]);
const markClicked = useCallback((url: string | undefined | null) => {
if (!url || clickedUrlsRef.current.has(url)) return; // unknown or already visited
// Optimistic, synchronous local update → the map pin recolours on click.
setClickedUrls((prev) => (prev.has(url) ? prev : new Set(prev).add(url)));
const uid = userIdRef.current;
if (!uid) return; // anonymous: recolour for the session, nothing to persist
pb.collection(CLICKED_LISTINGS_COLLECTION)
.create({ user: uid, url })
.catch(() => {
// Ignore duplicate-index conflicts and transient failures — local state already
// reflects the click, and a missed write only means it won't persist to the next visit.
});
}, []);
return { clickedUrls, markClicked };
}

View file

@ -17,6 +17,8 @@ import type {
import { import {
DENSITY_GRADIENT, DENSITY_GRADIENT,
DENSITY_GRADIENT_DARK, DENSITY_GRADIENT_DARK,
FILTERED_OUT_FILL,
FILTERED_OUT_LINE,
getEnumPaletteForFeature, getEnumPaletteForFeature,
getFeatureGradient, getFeatureGradient,
} from '../lib/consts'; } from '../lib/consts';
@ -125,11 +127,12 @@ export function useDeckLayers({
zoom, zoom,
isDark, isDark,
}); });
const { listingLayers, listingPopup, clearListingPopup } = useListingLayers({ const { listingLayers, listingPopup, clearListingPopup, clickedListingUrls, markListingClicked } =
listings: actualListings, useListingLayers({
zoom, listings: actualListings,
isDark, zoom,
}); isDark,
});
const { developmentLayers, developmentPopup, clearDevelopmentPopup } = useDevelopmentLayers({ const { developmentLayers, developmentPopup, clearDevelopmentPopup } = useDevelopmentLayers({
developments, developments,
zoom, zoom,
@ -350,7 +353,12 @@ export function useDeckLayers({
getHexagon: (d) => d.h3, getHexagon: (d) => d.h3,
getFillColor: (d) => { getFillColor: (d) => {
if ((d.count as number) <= 0) { if ((d.count as number) <= 0) {
return [0, 0, 0, 0] as [number, number, number, number]; // Filtered out: keep it on the map as a faint grey ghost (still
// clickable) rather than hiding it entirely.
return withColorOpacity(
FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
colorOpacityRef.current
);
} }
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current); const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
const dark = isDarkRef.current; const dark = isDarkRef.current;
@ -504,7 +512,12 @@ export function useDeckLayers({
getFillColor: (f) => { getFillColor: (f) => {
const d = f.properties; const d = f.properties;
if (d.count <= 0) { if (d.count <= 0) {
return [0, 0, 0, 0] as [number, number, number, number]; // Filtered out: keep it on the map as a faint grey ghost (still
// clickable) rather than hiding it entirely.
return withColorOpacity(
FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
colorOpacityRef.current
);
} }
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current); const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
const dark = isDarkRef.current; const dark = isDarkRef.current;
@ -587,7 +600,7 @@ export function useDeckLayers({
return [29, 228, 195, 255] as [number, number, number, number]; return [29, 228, 195, 255] as [number, number, number, number];
if (pc === hoveredPostcodeRef.current) if (pc === hoveredPostcodeRef.current)
return [29, 228, 195, 200] as [number, number, number, number]; return [29, 228, 195, 200] as [number, number, number, number];
if (f.properties.count <= 0) return [0, 0, 0, 0] as [number, number, number, number]; if (f.properties.count <= 0) return FILTERED_OUT_LINE[dark ? 'dark' : 'light'];
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [ return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
number, number,
number, number,
@ -599,7 +612,7 @@ export function useDeckLayers({
const pc = f.properties.postcode; const pc = f.properties.postcode;
if (pc === selectedPostcodeIdRef.current) return 4; if (pc === selectedPostcodeIdRef.current) return 4;
if (pc === hoveredPostcodeRef.current) return 2; if (pc === hoveredPostcodeRef.current) return 2;
if (f.properties.count <= 0) return 0; // Filtered-out polygons (count <= 0) keep the same 1px ghost border.
return 1; return 1;
}, },
lineWidthUnits: 'pixels', lineWidthUnits: 'pixels',
@ -749,6 +762,8 @@ export function useDeckLayers({
clearPopupInfo, clearPopupInfo,
listingPopup, listingPopup,
clearListingPopup, clearListingPopup,
clickedListingUrls,
markListingClicked,
developmentPopup, developmentPopup,
clearDevelopmentPopup, clearDevelopmentPopup,
hoverPosition, hoverPosition,

View file

@ -17,6 +17,15 @@ import {
getSpecificCrimeFilterKeyId, getSpecificCrimeFilterKeyId,
normalizeSpecificCrimeFilters, normalizeSpecificCrimeFilters,
} from '../lib/crime-filter'; } from '../lib/crime-filter';
import {
CRIME_SEVERITY_FILTER_NAMES,
createCrimeSeverityFilterKey,
getCrimeSeverityFeatureName,
getCrimeSeverityFilterKeyId,
getDefaultCrimeSeverityFeatureName,
normalizeCrimeSeverityFilters,
type CrimeSeverityFilterName,
} from '../lib/crime-severity-filter';
import { import {
ELECTION_VOTE_SHARE_FILTER_NAME, ELECTION_VOTE_SHARE_FILTER_NAME,
createElectionVoteShareFilterKey, createElectionVoteShareFilterKey,
@ -33,6 +42,22 @@ import {
getEthnicityFilterKeyId, getEthnicityFilterKeyId,
normalizeEthnicityFilters, normalizeEthnicityFilters,
} from '../lib/ethnicity-filter'; } from '../lib/ethnicity-filter';
import {
QUALIFICATIONS_FILTER_NAME,
createQualificationFilterKey,
getDefaultQualificationFeatureName,
getQualificationFeatureName,
getQualificationFilterKeyId,
normalizeQualificationFilters,
} from '../lib/qualification-filter';
import {
TENURE_FILTER_NAME,
createTenureFilterKey,
getDefaultTenureFeatureName,
getTenureFeatureName,
getTenureFilterKeyId,
normalizeTenureFilters,
} from '../lib/tenure-filter';
import { import {
POI_FILTER_NAMES, POI_FILTER_NAMES,
createPoiFilterKey, createPoiFilterKey,
@ -55,9 +80,15 @@ interface UseFiltersOptions {
function normalizeFilters(filters: FeatureFilters): FeatureFilters { function normalizeFilters(filters: FeatureFilters): FeatureFilters {
return normalizePoiDistanceFilters( return normalizePoiDistanceFilters(
normalizeEthnicityFilters( normalizeTenureFilters(
normalizeElectionVoteShareFilters( normalizeQualificationFilters(
normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters)) normalizeEthnicityFilters(
normalizeElectionVoteShareFilters(
normalizeCrimeSeverityFilters(
normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters))
)
)
)
) )
) )
); );
@ -67,8 +98,11 @@ function getBackendFeatureName(name: string): string {
return ( return (
getSchoolBackendFeatureName(name) ?? getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ?? getSpecificCrimeFeatureName(name) ??
getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ?? getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ?? getEthnicityFeatureName(name) ??
getQualificationFeatureName(name) ??
getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ?? getPoiDistanceFeatureName(name) ??
name name
); );
@ -140,12 +174,21 @@ export function useFilters({
const specificCrimeFilterIdRef = useRef( const specificCrimeFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getSpecificCrimeFilterKeyId) getNextNumericKeyId(initialFiltersRef.current!, getSpecificCrimeFilterKeyId)
); );
const crimeSeverityFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getCrimeSeverityFilterKeyId)
);
const electionVoteShareFilterIdRef = useRef( const electionVoteShareFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getElectionVoteShareFilterKeyId) getNextNumericKeyId(initialFiltersRef.current!, getElectionVoteShareFilterKeyId)
); );
const ethnicityFilterIdRef = useRef( const ethnicityFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getEthnicityFilterKeyId) getNextNumericKeyId(initialFiltersRef.current!, getEthnicityFilterKeyId)
); );
const qualificationFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getQualificationFilterKeyId)
);
const tenureFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getTenureFilterKeyId)
);
const poiDistanceFilterIdRef = useRef( const poiDistanceFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getPoiDistanceFilterKeyId) getNextNumericKeyId(initialFiltersRef.current!, getPoiDistanceFilterKeyId)
); );
@ -184,8 +227,11 @@ export function useFilters({
if ( if (
name !== SCHOOL_FILTER_NAME && name !== SCHOOL_FILTER_NAME &&
name !== SPECIFIC_CRIMES_FILTER_NAME && name !== SPECIFIC_CRIMES_FILTER_NAME &&
!CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) &&
name !== ELECTION_VOTE_SHARE_FILTER_NAME && name !== ELECTION_VOTE_SHARE_FILTER_NAME &&
name !== ETHNICITIES_FILTER_NAME && name !== ETHNICITIES_FILTER_NAME &&
name !== QUALIFICATIONS_FILTER_NAME &&
name !== TENURE_FILTER_NAME &&
!POI_FILTER_NAMES.includes(name as PoiFilterName) && !POI_FILTER_NAMES.includes(name as PoiFilterName) &&
!meta !meta
) { ) {
@ -201,8 +247,11 @@ export function useFilters({
const addsNewKey = const addsNewKey =
name === SCHOOL_FILTER_NAME || name === SCHOOL_FILTER_NAME ||
name === SPECIFIC_CRIMES_FILTER_NAME || name === SPECIFIC_CRIMES_FILTER_NAME ||
CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) ||
name === ELECTION_VOTE_SHARE_FILTER_NAME || name === ELECTION_VOTE_SHARE_FILTER_NAME ||
name === ETHNICITIES_FILTER_NAME || name === ETHNICITIES_FILTER_NAME ||
name === QUALIFICATIONS_FILTER_NAME ||
name === TENURE_FILTER_NAME ||
POI_FILTER_NAMES.includes(name as PoiFilterName) || POI_FILTER_NAMES.includes(name as PoiFilterName) ||
!(name in current); !(name in current);
if (addsNewKey && Object.keys(current).length >= limit) { if (addsNewKey && Object.keys(current).length >= limit) {
@ -247,6 +296,29 @@ export function useFilters({
], ],
}; };
} }
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
const severityFilterName = name as CrimeSeverityFilterName;
const defaultSeverityFeatureName = getDefaultCrimeSeverityFeatureName(
features,
severityFilterName
);
const defaultSeverityFeature = defaultSeverityFeatureName
? features.find((feature) => feature.name === defaultSeverityFeatureName)
: undefined;
if (!defaultSeverityFeatureName) return prev;
return {
...prev,
[createCrimeSeverityFilterKey(
severityFilterName,
defaultSeverityFeatureName,
crimeSeverityFilterIdRef.current++
)]: [
defaultSeverityFeature?.histogram?.min ?? defaultSeverityFeature?.min ?? 0,
defaultSeverityFeature?.histogram?.max ?? defaultSeverityFeature?.max ?? 100,
],
};
}
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) { if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
const defaultVoteShareFeatureName = getDefaultElectionVoteShareFeatureName(features); const defaultVoteShareFeatureName = getDefaultElectionVoteShareFeatureName(features);
const defaultVoteShareFeature = defaultVoteShareFeatureName const defaultVoteShareFeature = defaultVoteShareFeatureName
@ -281,6 +353,41 @@ export function useFilters({
], ],
}; };
} }
if (name === QUALIFICATIONS_FILTER_NAME) {
const defaultQualificationFeatureName = getDefaultQualificationFeatureName(features);
const defaultQualificationFeature = defaultQualificationFeatureName
? features.find((feature) => feature.name === defaultQualificationFeatureName)
: undefined;
if (!defaultQualificationFeatureName) return prev;
return {
...prev,
[createQualificationFilterKey(
defaultQualificationFeatureName,
qualificationFilterIdRef.current++
)]: [
defaultQualificationFeature?.histogram?.min ?? defaultQualificationFeature?.min ?? 0,
defaultQualificationFeature?.histogram?.max ??
defaultQualificationFeature?.max ??
100,
],
};
}
if (name === TENURE_FILTER_NAME) {
const defaultTenureFeatureName = getDefaultTenureFeatureName(features);
const defaultTenureFeature = defaultTenureFeatureName
? features.find((feature) => feature.name === defaultTenureFeatureName)
: undefined;
if (!defaultTenureFeatureName) return prev;
return {
...prev,
[createTenureFilterKey(defaultTenureFeatureName, tenureFilterIdRef.current++)]: [
defaultTenureFeature?.histogram?.min ?? defaultTenureFeature?.min ?? 0,
defaultTenureFeature?.histogram?.max ?? defaultTenureFeature?.max ?? 100,
],
};
}
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) { if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
const poiFilterName = name as PoiFilterName; const poiFilterName = name as PoiFilterName;
const defaultPoiFeatureName = getDefaultPoiFilterFeatureName(features, poiFilterName); const defaultPoiFeatureName = getDefaultPoiFilterFeatureName(features, poiFilterName);
@ -381,6 +488,23 @@ export function useFilters({
if (replaced) return normalizeFilters(next); if (replaced) return normalizeFilters(next);
} }
const crimeSeverityKeyId = getCrimeSeverityFilterKeyId(name);
if (crimeSeverityKeyId != null) {
let replaced = false;
const next: FeatureFilters = {};
for (const [existingName, existingValue] of Object.entries(prev)) {
if (getCrimeSeverityFilterKeyId(existingName) === crimeSeverityKeyId) {
if (!replaced) {
next[name] = value;
replaced = true;
}
continue;
}
next[existingName] = existingValue;
}
if (replaced) return normalizeFilters(next);
}
const ethnicityKeyId = getEthnicityFilterKeyId(name); const ethnicityKeyId = getEthnicityFilterKeyId(name);
if (ethnicityKeyId != null) { if (ethnicityKeyId != null) {
let replaced = false; let replaced = false;
@ -398,6 +522,40 @@ export function useFilters({
if (replaced) return normalizeFilters(next); if (replaced) return normalizeFilters(next);
} }
const qualificationKeyId = getQualificationFilterKeyId(name);
if (qualificationKeyId != null) {
let replaced = false;
const next: FeatureFilters = {};
for (const [existingName, existingValue] of Object.entries(prev)) {
if (getQualificationFilterKeyId(existingName) === qualificationKeyId) {
if (!replaced) {
next[name] = value;
replaced = true;
}
continue;
}
next[existingName] = existingValue;
}
if (replaced) return normalizeFilters(next);
}
const tenureKeyId = getTenureFilterKeyId(name);
if (tenureKeyId != null) {
let replaced = false;
const next: FeatureFilters = {};
for (const [existingName, existingValue] of Object.entries(prev)) {
if (getTenureFilterKeyId(existingName) === tenureKeyId) {
if (!replaced) {
next[name] = value;
replaced = true;
}
continue;
}
next[existingName] = existingValue;
}
if (replaced) return normalizeFilters(next);
}
const electionVoteShareKeyId = getElectionVoteShareFilterKeyId(name); const electionVoteShareKeyId = getElectionVoteShareFilterKeyId(name);
if (electionVoteShareKeyId != null) { if (electionVoteShareKeyId != null) {
let replaced = false; let replaced = false;

View file

@ -10,7 +10,14 @@ import type {
HexagonPropertiesResponse, HexagonPropertiesResponse,
HexagonStatsResponse, HexagonStatsResponse,
} from '../types'; } from '../types';
import { buildFilterString, apiUrl, assertOk, logNonAbortError, authHeaders } from '../lib/api'; import {
buildFilterString,
apiUrl,
assertOk,
logNonAbortError,
authHeaders,
fetchCrimeRecords,
} from '../lib/api';
import { findOverlappingSelectableHexagon } from '../lib/h3-selection'; import { findOverlappingSelectableHexagon } from '../lib/h3-selection';
import { SMALLEST_VISIBLE_HEXAGON_RESOLUTION } from '../lib/consts'; import { SMALLEST_VISIBLE_HEXAGON_RESOLUTION } from '../lib/consts';
import type { TravelTimeEntry } from './useTravelTime'; import type { TravelTimeEntry } from './useTravelTime';
@ -327,6 +334,23 @@ export function useHexagonSelection({
] ]
); );
// Lazily fetch a page of individual crimes for the current selection. The
// records are an attribute of the area (filter-independent), so this carries
// no filter string — only the selection identity and share code.
const loadCrimeRecords = useCallback(
(offset: number) => {
if (!selectedHexagon) {
return Promise.resolve({ records: [], total: 0, offset: 0, truncated: false });
}
const selection =
selectedHexagon.type === 'postcode'
? { postcode: selectedHexagon.id }
: { h3: selectedHexagon.id, resolution: selectedHexagon.resolution };
return fetchCrimeRecords(selection, offset, shareCode);
},
[selectedHexagon, shareCode]
);
const handleHexagonClick = useCallback( const handleHexagonClick = useCallback(
(id: string, isPostcode = false, geometry?: PostcodeGeometry) => { (id: string, isPostcode = false, geometry?: PostcodeGeometry) => {
if (selectedHexagon?.id === id) { if (selectedHexagon?.id === id) {
@ -806,6 +830,7 @@ export function useHexagonSelection({
handlePropertiesTabClick, handlePropertiesTabClick,
handleLoadMoreProperties, handleLoadMoreProperties,
handleCloseSelection, handleCloseSelection,
loadCrimeRecords,
selectedPostcodeGeometry, selectedPostcodeGeometry,
handleLocationSearch, handleLocationSearch,
handleCurrentLocationSearch, handleCurrentLocationSearch,

View file

@ -1,8 +1,10 @@
import { useState, useCallback } from 'react'; import { useState, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { apiUrl, authHeaders, assertOk } from '../lib/api'; import { apiUrl, authHeaders, assertOk } from '../lib/api';
import { trackEvent } from '../lib/analytics'; import { trackEvent } from '../lib/analytics';
export function useLicense() { export function useLicense() {
const { t } = useTranslation();
const [checkingOut, setCheckingOut] = useState(false); const [checkingOut, setCheckingOut] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
@ -25,13 +27,13 @@ export function useLicense() {
window.location.href = data.url; window.location.href = data.url;
} }
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Checkout failed'; const msg = err instanceof Error ? err.message : t('upgrade.checkoutFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setCheckingOut(false); setCheckingOut(false);
} }
}, []); }, [t]);
return { startCheckout, checkingOut, error }; return { startCheckout, checkingOut, error };
} }

View file

@ -5,6 +5,13 @@ import Supercluster from 'supercluster';
import type { ActualListing } from '../types'; import type { ActualListing } from '../types';
import { trackEvent } from '../lib/analytics'; import { trackEvent } from '../lib/analytics';
import { useClickedListings } from './useClickedListings';
/** Default red pin vs. the violet used once a user has opened that listing. */
const LISTING_PIN_FILL: [number, number, number, number] = [231, 76, 60, 240];
const LISTING_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 240];
const EXPANDED_PIN_FILL: [number, number, number, number] = [231, 76, 60, 245];
const EXPANDED_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 245];
const PRICE_LABEL_MIN_ZOOM = 14; const PRICE_LABEL_MIN_ZOOM = 14;
const ADDRESS_LABEL_MIN_ZOOM = 16; const ADDRESS_LABEL_MIN_ZOOM = 16;
@ -113,6 +120,7 @@ function spiderfyPosition(
export function useListingLayers({ listings, zoom, isDark }: UseListingLayersProps) { export function useListingLayers({ listings, zoom, isDark }: UseListingLayersProps) {
const [popupInfo, setPopupInfo] = useState<ListingPopupInfo | null>(null); const [popupInfo, setPopupInfo] = useState<ListingPopupInfo | null>(null);
const [selectedCluster, setSelectedCluster] = useState<ListingClusterPoint | null>(null); const [selectedCluster, setSelectedCluster] = useState<ListingClusterPoint | null>(null);
const { clickedUrls, markClicked } = useClickedListings();
useEffect(() => { useEffect(() => {
// Each refetch returns a fresh listings array and rebuilds the cluster index, so a // Each refetch returns a fresh listings array and rebuilds the cluster index, so a
@ -212,12 +220,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
[clearUnlockedPopup] [clearUnlockedPopup]
); );
const handleClick = useCallback((info: PickingInfo<ActualListing>) => { const handleClick = useCallback(
const url = info.object?.listing_url; (info: PickingInfo<ActualListing>) => {
if (!url) return; const url = info.object?.listing_url;
trackEvent('Actual Listing Click', { url }); if (!url) return;
window.open(url, '_blank', 'noopener,noreferrer'); markClicked(url);
}, []); trackEvent('Actual Listing Click', { url });
window.open(url, '_blank', 'noopener,noreferrer');
},
[markClicked]
);
const handleHoverRef = useRef(handleHover); const handleHoverRef = useRef(handleHover);
handleHoverRef.current = handleHover; handleHoverRef.current = handleHover;
@ -244,12 +256,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
[clearUnlockedPopup] [clearUnlockedPopup]
); );
const handleExpandedClick = useCallback((info: PickingInfo<ExpandedListingMarker>) => { const handleExpandedClick = useCallback(
const url = info.object?.listing.listing_url; (info: PickingInfo<ExpandedListingMarker>) => {
if (!url) return; const url = info.object?.listing.listing_url;
trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' }); if (!url) return;
window.open(url, '_blank', 'noopener,noreferrer'); markClicked(url);
}, []); trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' });
window.open(url, '_blank', 'noopener,noreferrer');
},
[markClicked]
);
const handleExpandedHoverRef = useRef(handleExpandedHover); const handleExpandedHoverRef = useRef(handleExpandedHover);
handleExpandedHoverRef.current = handleExpandedHover; handleExpandedHoverRef.current = handleExpandedHover;
@ -345,7 +361,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
getPosition: (d) => [d.lon, d.lat], getPosition: (d) => [d.lon, d.lat],
getRadius: 7, getRadius: 7,
radiusUnits: 'pixels', radiusUnits: 'pixels',
getFillColor: [231, 76, 60, 240], getFillColor: (d) =>
clickedUrls.has(d.listing_url) ? LISTING_PIN_VISITED_FILL : LISTING_PIN_FILL,
getLineColor: [255, 255, 255, 255], getLineColor: [255, 255, 255, 255],
getLineWidth: 1.5, getLineWidth: 1.5,
lineWidthUnits: 'pixels', lineWidthUnits: 'pixels',
@ -355,8 +372,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
highlightColor: [29, 228, 195, 220], highlightColor: [29, 228, 195, 220],
onHover: stableHover, onHover: stableHover,
onClick: stableClick, onClick: stableClick,
updateTriggers: { getFillColor: clickedUrls },
}), }),
[visibleListings, stableHover, stableClick] [visibleListings, clickedUrls, stableHover, stableClick]
); );
const clusterShadowLayer = useMemo( const clusterShadowLayer = useMemo(
@ -441,7 +459,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
getPosition: (d) => [d.lng, d.lat], getPosition: (d) => [d.lng, d.lat],
getRadius: 6, getRadius: 6,
radiusUnits: 'pixels', radiusUnits: 'pixels',
getFillColor: [231, 76, 60, 245], getFillColor: (d) =>
clickedUrls.has(d.listing.listing_url) ? EXPANDED_PIN_VISITED_FILL : EXPANDED_PIN_FILL,
getLineColor: [255, 255, 255, 255], getLineColor: [255, 255, 255, 255],
getLineWidth: 1.5, getLineWidth: 1.5,
lineWidthUnits: 'pixels', lineWidthUnits: 'pixels',
@ -451,8 +470,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
highlightColor: [29, 228, 195, 220], highlightColor: [29, 228, 195, 220],
onHover: stableExpandedHover, onHover: stableExpandedHover,
onClick: stableExpandedClick, onClick: stableExpandedClick,
updateTriggers: { getFillColor: clickedUrls },
}), }),
[expandedListings, stableExpandedHover, stableExpandedClick] [expandedListings, clickedUrls, stableExpandedHover, stableExpandedClick]
); );
const priceLabelLayer = useMemo(() => { const priceLabelLayer = useMemo(() => {
@ -544,5 +564,11 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
setSelectedCluster(null); setSelectedCluster(null);
}, []); }, []);
return { listingLayers, listingPopup: popupInfo, clearListingPopup }; return {
listingLayers,
listingPopup: popupInfo,
clearListingPopup,
clickedListingUrls: clickedUrls,
markListingClicked: markClicked,
};
} }

View file

@ -18,8 +18,11 @@ import {
} from '../lib/api'; } from '../lib/api';
import { getSchoolBackendFeatureName } from '../lib/school-filter'; import { getSchoolBackendFeatureName } from '../lib/school-filter';
import { getSpecificCrimeFeatureName } from '../lib/crime-filter'; import { getSpecificCrimeFeatureName } from '../lib/crime-filter';
import { getCrimeSeverityFeatureName } from '../lib/crime-severity-filter';
import { getElectionVoteShareFeatureName } from '../lib/election-filter'; import { getElectionVoteShareFeatureName } from '../lib/election-filter';
import { getEthnicityFeatureName } from '../lib/ethnicity-filter'; import { getEthnicityFeatureName } from '../lib/ethnicity-filter';
import { getQualificationFeatureName } from '../lib/qualification-filter';
import { getTenureFeatureName } from '../lib/tenure-filter';
import { getPoiDistanceFeatureName } from '../lib/poi-distance-filter'; import { getPoiDistanceFeatureName } from '../lib/poi-distance-filter';
import { POSTCODE_ZOOM_THRESHOLD } from '../lib/consts'; import { POSTCODE_ZOOM_THRESHOLD } from '../lib/consts';
import { COLOR_RANGE_LOW_PERCENTILE, COLOR_RANGE_HIGH_PERCENTILE } from '../lib/consts'; import { COLOR_RANGE_LOW_PERCENTILE, COLOR_RANGE_HIGH_PERCENTILE } from '../lib/consts';
@ -119,8 +122,11 @@ export function useMapData({
(name: string) => (name: string) =>
getSchoolBackendFeatureName(name) ?? getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ?? getSpecificCrimeFeatureName(name) ??
getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ?? getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ?? getEthnicityFeatureName(name) ??
getQualificationFeatureName(name) ??
getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ?? getPoiDistanceFeatureName(name) ??
name, name,
[] []

View file

@ -1,9 +1,19 @@
import { act, renderHook } from '@testing-library/react'; import { act, renderHook } from '@testing-library/react';
import { describe, expect, it } from 'vitest'; import { describe, expect, it, vi } from 'vitest';
import type { POI } from '../types'; import type { POI } from '../types';
import { usePoiLayers } from './usePoiLayers'; import { usePoiLayers } from './usePoiLayers';
vi.mock('react-i18next', () => ({
useTranslation: () => ({
t: (key: string) => {
if (key === 'common.places') return 'places';
if (key === 'map.poi.zoomInToSeeDetails') return 'Zoom in to see details';
return key;
},
}),
}));
const supermarket: POI = { const supermarket: POI = {
id: 'poi-1', id: 'poi-1',
name: 'Market Hall', name: 'Market Hall',

View file

@ -1,4 +1,5 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import type { PickingInfo } from '@deck.gl/core'; import type { PickingInfo } from '@deck.gl/core';
import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers'; import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers';
import Supercluster from 'supercluster'; import Supercluster from 'supercluster';
@ -65,6 +66,7 @@ function getPoiIconSize(poi: POI): number {
} }
export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) { export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
const { t } = useTranslation();
const [popupInfo, setPopupInfo] = useState<PopupInfo | null>(null); const [popupInfo, setPopupInfo] = useState<PopupInfo | null>(null);
// Dismiss a lingering hover/cluster popup once the POIs behind it are gone — e.g. // Dismiss a lingering hover/cluster popup once the POIs behind it are gone — e.g.
@ -108,8 +110,8 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
setPopupInfo({ setPopupInfo({
x: info.x, x: info.x,
y: info.y, y: info.y,
name: `${info.object.count} places`, name: `${info.object.count} ${t('common.places')}`,
category: 'Zoom in to see details', category: t('map.poi.zoomInToSeeDetails'),
group: '', group: '',
emoji: '', emoji: '',
id: '', id: '',
@ -119,7 +121,7 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
} else { } else {
setPopupInfo(null); setPopupInfo(null);
} }
}, []); }, [t]);
const handleClusterHoverRef = useRef(handleClusterHover); const handleClusterHoverRef = useRef(handleClusterHover);
handleClusterHoverRef.current = handleClusterHover; handleClusterHoverRef.current = handleClusterHover;

View file

@ -0,0 +1,80 @@
import { cleanup, fireEvent, render } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { useState } from 'react';
import { useRevealOnExpand } from './useRevealOnExpand';
function GroupList() {
const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
const [expanded, setExpanded] = useState(false);
return (
<div ref={setContainer} data-testid="container">
<div ref={registerGroup('g1')} data-testid="group">
<button
data-testid="toggle"
onClick={() => {
const willExpand = !expanded;
setExpanded(willExpand);
onToggle('g1', willExpand);
}}
>
toggle
</button>
{expanded && <div style={{ height: 1000 }} />}
</div>
<div style={{ height: 2000 }} />
</div>
);
}
describe('useRevealOnExpand', () => {
const original = {
resizeObserver: globalThis.ResizeObserver,
scrollTo: HTMLElement.prototype.scrollTo,
getBoundingClientRect: HTMLElement.prototype.getBoundingClientRect,
};
beforeEach(() => {
globalThis.ResizeObserver = class {
observe() {}
unobserve() {}
disconnect() {}
} as unknown as typeof ResizeObserver;
HTMLElement.prototype.scrollTo = vi.fn();
// The opened group's bottom (800) sits below the container's bottom (500),
// so it needs a 300px downward scroll to come fully into view.
HTMLElement.prototype.getBoundingClientRect = function (this: HTMLElement) {
const id = this.dataset?.testid;
const bottom = id === 'group' ? 800 : id === 'container' ? 500 : 0;
return { bottom } as DOMRect;
};
});
afterEach(() => {
cleanup();
globalThis.ResizeObserver = original.resizeObserver;
HTMLElement.prototype.scrollTo = original.scrollTo;
HTMLElement.prototype.getBoundingClientRect = original.getBoundingClientRect;
vi.restoreAllMocks();
});
it('scrolls a freshly expanded group into view', () => {
const view = render(<GroupList />);
const scrollTo = view.getByTestId('container').scrollTo as ReturnType<typeof vi.fn>;
fireEvent.click(view.getByTestId('toggle'));
expect(scrollTo).toHaveBeenCalledWith({ top: 300 });
});
it('does not scroll when a group is collapsed', () => {
const view = render(<GroupList />);
const scrollTo = view.getByTestId('container').scrollTo as ReturnType<typeof vi.fn>;
fireEvent.click(view.getByTestId('toggle')); // expand
scrollTo.mockClear();
fireEvent.click(view.getByTestId('toggle')); // collapse
expect(scrollTo).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,97 @@
import { useCallback, useEffect, useRef, useState } from 'react';
/**
* Auto-scrolls a freshly expanded collapsible group into view inside a scroll
* container, so opening a group near the bottom reveals its content without
* manual scrolling. The (sticky) group header stays pinned at the top.
*
* Wiring:
* const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
* <div ref={setContainer} className="overflow-y-auto">
* {groups.map((g) => (
* <div key={g.name} ref={registerGroup(g.name)}>
* <Header
* onToggle={() => {
* const willExpand = !isExpanded(g.name);
* toggle(g.name);
* onToggle(g.name, willExpand);
* }}
* />
* {isExpanded(g.name) && <Body />}
* </div>
* ))}
* </div>
*
* When the container element is also driven by another ref (e.g. the callback
* ref from useRetainedScrollTop), compose them by calling both inside a single
* callback ref.
*/
export function useRevealOnExpand<T extends HTMLElement = HTMLDivElement>() {
const containerRef = useRef<T | null>(null);
const groupRefs = useRef<Map<string, T>>(new Map());
const groupRefCallbacks = useRef<Map<string, (node: T | null) => void>>(new Map());
// A fresh object each toggle so re-expanding the same group still re-triggers.
const [groupToReveal, setGroupToReveal] = useState<{ name: string } | null>(null);
const setContainer = useCallback((node: T | null) => {
containerRef.current = node;
}, []);
// Returns a stable ref callback per group name so React doesn't churn the map
// (cleanup + re-register) on every render.
const registerGroup = useCallback((name: string) => {
const existing = groupRefCallbacks.current.get(name);
if (existing) return existing;
const cb = (node: T | null) => {
if (node) groupRefs.current.set(name, node);
else groupRefs.current.delete(name);
};
groupRefCallbacks.current.set(name, cb);
return cb;
}, []);
// Call from a group's toggle handler: reveals the group when it is being
// expanded, and cancels any pending reveal when it is being collapsed.
const onToggle = useCallback((name: string, willExpand: boolean) => {
setGroupToReveal(willExpand ? { name } : null);
}, []);
useEffect(() => {
if (!groupToReveal) return;
const el = groupRefs.current.get(groupToReveal.name);
const container = containerRef.current;
if (!el || !container) return;
// Scroll down just enough to bring the bottom of the opened group into view.
// For groups taller than the pane this scrolls past the header, but it stays
// visible via its sticky positioning. Never scroll up (group already in view).
const alignBottom = () => {
const delta = el.getBoundingClientRect().bottom - container.getBoundingClientRect().bottom;
if (delta <= 1) return;
container.scrollTo({ top: container.scrollTop + delta });
};
alignBottom();
// The group's charts and async data (e.g. nearby stations) can grow its
// height after this first pass, so keep the bottom pinned as it settles.
// Stop once the user scrolls or after a short grace period so we never fight
// a deliberate scroll.
let timer = 0;
const observer = new ResizeObserver(alignBottom);
const stop = () => {
observer.disconnect();
container.removeEventListener('wheel', stop);
container.removeEventListener('touchmove', stop);
window.clearTimeout(timer);
};
observer.observe(el);
container.addEventListener('wheel', stop, { passive: true });
container.addEventListener('touchmove', stop, { passive: true });
timer = window.setTimeout(stop, 1500);
return stop;
}, [groupToReveal]);
return { setContainer, registerGroup, onToggle };
}

View file

@ -1,4 +1,5 @@
import { useState, useCallback, useRef, useEffect } from 'react'; import { useState, useCallback, useRef, useEffect } from 'react';
import { useTranslation } from 'react-i18next';
import pb from '../lib/pocketbase'; import pb from '../lib/pocketbase';
import { apiUrl, authHeaders } from '../lib/api'; import { apiUrl, authHeaders } from '../lib/api';
import { trackEvent } from '../lib/analytics'; import { trackEvent } from '../lib/analytics';
@ -24,6 +25,7 @@ function nextPollDelay(attempt: number): number {
} }
export function useSavedSearches(userId: string | null) { export function useSavedSearches(userId: string | null) {
const { t } = useTranslation();
const [searches, setSearches] = useState<SavedSearch[]>([]); const [searches, setSearches] = useState<SavedSearch[]>([]);
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
@ -127,11 +129,11 @@ export function useSavedSearches(userId: string | null) {
stopPolling(); stopPolling();
} }
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : 'Failed to load searches'); setError(err instanceof Error ? err.message : t('savedPage.loadFailed'));
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, [userId, fetchRecords, startPolling, stopPolling]); }, [userId, fetchRecords, startPolling, stopPolling, t]);
const saveSearch = useCallback( const saveSearch = useCallback(
async (name: string, paramsOverride?: string) => { async (name: string, paramsOverride?: string) => {
@ -169,14 +171,14 @@ export function useSavedSearches(userId: string | null) {
console.warn('Background screenshot failed:', err); console.warn('Background screenshot failed:', err);
}); });
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to save search'; const msg = err instanceof Error ? err.message : t('savedPage.saveFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setSaving(false); setSaving(false);
} }
}, },
[userId, fetchSearches] [userId, fetchSearches, t]
); );
const deleteSearch = useCallback(async (id: string) => { const deleteSearch = useCallback(async (id: string) => {
@ -186,27 +188,27 @@ export function useSavedSearches(userId: string | null) {
trackEvent('Search Delete'); trackEvent('Search Delete');
setSearches((prev) => prev.filter((s) => s.id !== id)); setSearches((prev) => prev.filter((s) => s.id !== id));
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : 'Failed to delete search'); setError(err instanceof Error ? err.message : t('savedPage.deleteFailed'));
} }
}, []); }, [t]);
const updateSearchNotes = useCallback(async (id: string, notes: string) => { const updateSearchNotes = useCallback(async (id: string, notes: string) => {
try { try {
await pb.collection('saved_searches').update(id, { notes }); await pb.collection('saved_searches').update(id, { notes });
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, notes } : s))); setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, notes } : s)));
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : 'Failed to update notes'); setError(err instanceof Error ? err.message : t('savedPage.updateNotesFailed'));
} }
}, []); }, [t]);
const updateSearchName = useCallback(async (id: string, name: string) => { const updateSearchName = useCallback(async (id: string, name: string) => {
try { try {
await pb.collection('saved_searches').update(id, { name }); await pb.collection('saved_searches').update(id, { name });
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, name } : s))); setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, name } : s)));
} catch (err) { } catch (err) {
setError(err instanceof Error ? err.message : 'Failed to update name'); setError(err instanceof Error ? err.message : t('savedPage.updateNameFailed'));
} }
}, []); }, [t]);
const updateSearchParams = useCallback( const updateSearchParams = useCallback(
async (id: string, params: string) => { async (id: string, params: string) => {
@ -238,14 +240,14 @@ export function useSavedSearches(userId: string | null) {
console.warn('Background screenshot failed:', err); console.warn('Background screenshot failed:', err);
}); });
} catch (err) { } catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to update search'; const msg = err instanceof Error ? err.message : t('savedPage.updateFailed');
setError(msg); setError(msg);
throw err; throw err;
} finally { } finally {
setSaving(false); setSaving(false);
} }
}, },
[userId, fetchSearches] [userId, fetchSearches, t]
); );
return { return {

View file

@ -50,6 +50,13 @@ export function useTutorial(initialLoading: boolean, isMobile: boolean, blocked
placement: 'left' as const, placement: 'left' as const,
skipBeacon: true, skipBeacon: true,
}, },
{
target: '[data-tutorial="overlays-button"]',
title: t('tutorial.step7Title'),
content: t('tutorial.step7Content'),
placement: 'left' as const,
skipBeacon: true,
},
], ],
[t] [t]
); );

View file

@ -1,4 +1,5 @@
import { createRoot } from 'react-dom/client'; import { createRoot } from 'react-dom/client';
import { useTranslation } from 'react-i18next';
import App from './App'; import App from './App';
import { i18nReady } from './i18n'; import { i18nReady } from './i18n';
import { BugsinkErrorBoundary, initBugsink } from './lib/bugsink'; import { BugsinkErrorBoundary, initBugsink } from './lib/bugsink';
@ -14,12 +15,17 @@ if (!container) {
const root = container; const root = container;
function AppErrorFallback() { function AppErrorFallback() {
const { t } = useTranslation();
return ( return (
<div className="flex min-h-screen items-center justify-center bg-warm-50 px-6 text-center text-warm-900 dark:bg-navy-950 dark:text-warm-100"> <div className="flex min-h-screen items-center justify-center bg-warm-50 px-6 text-center text-warm-900 dark:bg-navy-950 dark:text-warm-100">
<div> <div>
<h1 className="text-xl font-semibold">Something went wrong</h1> <h1 className="text-xl font-semibold">
{t('errors.appCrashTitle', { defaultValue: 'Something went wrong' })}
</h1>
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300"> <p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
Refresh the page to try again. {t('errors.appCrashBody', {
defaultValue: 'Refresh the page to try again.',
})}
</p> </p>
</div> </div>
</div> </div>

View file

@ -6,6 +6,8 @@ import { createSchoolFilterKey } from './school-filter';
import { createSpecificCrimeFilterKey } from './crime-filter'; import { createSpecificCrimeFilterKey } from './crime-filter';
import { createElectionVoteShareFilterKey } from './election-filter'; import { createElectionVoteShareFilterKey } from './election-filter';
import { createEthnicityFilterKey } from './ethnicity-filter'; import { createEthnicityFilterKey } from './ethnicity-filter';
import { createQualificationFilterKey } from './qualification-filter';
import { createTenureFilterKey } from './tenure-filter';
import { import {
POI_COUNT_2KM_FILTER_NAME, POI_COUNT_2KM_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME, TRANSPORT_DISTANCE_FILTER_NAME,
@ -98,19 +100,19 @@ describe('api utilities', () => {
it('serializes specific crime filters using their selected backend crime feature', () => { it('serializes specific crime filters using their selected backend crime feature', () => {
const features: FeatureMeta[] = [ const features: FeatureMeta[] = [
{ name: 'Burglary (avg/yr)', type: 'numeric', min: 0, max: 20 }, { name: 'Burglary (/yr, 7y)', type: 'numeric', min: 0, max: 20 },
{ name: 'Vehicle crime (avg/yr)', type: 'numeric', min: 0, max: 30 }, { name: 'Vehicle crime (/yr, 7y)', type: 'numeric', min: 0, max: 30 },
]; ];
expect( expect(
buildFilterString( buildFilterString(
{ {
[createSpecificCrimeFilterKey('Burglary (avg/yr)', 1)]: [0, 5], [createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1)]: [0, 5],
[createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2)]: [1, 10], [createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2)]: [1, 10],
}, },
features features
) )
).toBe('Burglary (avg/yr):0:5;;Vehicle crime (avg/yr):1:10'); ).toBe('Burglary (/yr, 7y):0:5;;Vehicle crime (/yr, 7y):1:10');
}); });
it('serializes election vote-share filters using their selected backend party feature', () => { it('serializes election vote-share filters using their selected backend party feature', () => {
@ -144,6 +146,40 @@ describe('api utilities', () => {
).toBe('% White:20:80'); ).toBe('% White:20:80');
}); });
it('serializes qualification filters using their selected backend band feature', () => {
const features: FeatureMeta[] = [
{ name: '% Degree or higher', type: 'numeric', min: 0, max: 100 },
{ name: '% No qualifications', type: 'numeric', min: 0, max: 100 },
];
expect(
buildFilterString(
{
[createQualificationFilterKey('% Degree or higher', 1)]: [20, 60],
[createQualificationFilterKey('% No qualifications', 2)]: [0, 25],
},
features
)
).toBe('% Degree or higher:20:60;;% No qualifications:0:25');
});
it('serializes tenure filters using their selected backend band feature', () => {
const features: FeatureMeta[] = [
{ name: '% Owner occupied', type: 'numeric', min: 0, max: 100 },
{ name: '% Private rent', type: 'numeric', min: 0, max: 100 },
];
expect(
buildFilterString(
{
[createTenureFilterKey('% Owner occupied', 1)]: [20, 60],
[createTenureFilterKey('% Private rent', 2)]: [0, 25],
},
features
)
).toBe('% Owner occupied:20:60;;% Private rent:0:25');
});
it('serializes amenity distance filters using their selected backend feature', () => { it('serializes amenity distance filters using their selected backend feature', () => {
const features: FeatureMeta[] = [ const features: FeatureMeta[] = [
{ name: 'Distance to nearest amenity (Park) (km)', type: 'numeric', min: 0, max: 2 }, { name: 'Distance to nearest amenity (Park) (km)', type: 'numeric', min: 0, max: 2 },

View file

@ -1,10 +1,13 @@
import type { FeatureMeta, FeatureFilters } from '../types'; import type { FeatureMeta, FeatureFilters, CrimeRecordsResponse } from '../types';
import { INITIAL_RETRY_MS, MAX_RETRY_MS } from './consts'; import { INITIAL_RETRY_MS, MAX_RETRY_MS } from './consts';
import pb from './pocketbase'; import pb from './pocketbase';
import { getSchoolBackendFeatureName } from './school-filter'; import { getSchoolBackendFeatureName } from './school-filter';
import { getSpecificCrimeFeatureName } from './crime-filter'; import { getSpecificCrimeFeatureName } from './crime-filter';
import { getCrimeSeverityFeatureName } from './crime-severity-filter';
import { getElectionVoteShareFeatureName } from './election-filter'; import { getElectionVoteShareFeatureName } from './election-filter';
import { getEthnicityFeatureName } from './ethnicity-filter'; import { getEthnicityFeatureName } from './ethnicity-filter';
import { getQualificationFeatureName } from './qualification-filter';
import { getTenureFeatureName } from './tenure-filter';
import { getPoiDistanceFeatureName } from './poi-distance-filter'; import { getPoiDistanceFeatureName } from './poi-distance-filter';
const SCREENSHOT_LANGUAGES = new Set(['en', 'fr', 'de', 'zh', 'hi', 'hu']); const SCREENSHOT_LANGUAGES = new Set(['en', 'fr', 'de', 'zh', 'hi', 'hu']);
@ -61,6 +64,7 @@ const DEMO_GATED_ENDPOINTS = new Set([
'postcode-stats', 'postcode-stats',
'postcode-properties', 'postcode-properties',
'hexagon-properties', 'hexagon-properties',
'crime-records',
'journey', 'journey',
'actual-listings', 'actual-listings',
'developments', 'developments',
@ -186,6 +190,26 @@ export async function shortenUrl(params: string, language?: string): Promise<str
return `${window.location.origin}${data.url}`; return `${window.location.origin}${data.url}`;
} }
/** Fetch one page of individual crime records for a hexagon or a postcode. The
* list is independent of property filters, so no filter string is sent. */
export async function fetchCrimeRecords(
selection: { h3: string; resolution: number } | { postcode: string },
offset: number,
shareCode?: string
): Promise<CrimeRecordsResponse> {
const params = new URLSearchParams({ offset: offset.toString() });
if ('postcode' in selection) {
params.set('postcode', selection.postcode);
} else {
params.set('h3', selection.h3);
params.set('resolution', selection.resolution.toString());
}
if (shareCode) params.set('share', shareCode);
const res = await fetch(apiUrl('crime-records', params), authHeaders());
assertOk(res, 'crime-records');
return res.json();
}
export function buildFilterString( export function buildFilterString(
filters: FeatureFilters, filters: FeatureFilters,
features: FeatureMeta[], features: FeatureMeta[],
@ -200,8 +224,11 @@ export function buildFilterString(
const backendName = const backendName =
getSchoolBackendFeatureName(name) ?? getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ?? getSpecificCrimeFeatureName(name) ??
getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ?? getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ?? getEthnicityFeatureName(name) ??
getQualificationFeatureName(name) ??
getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ?? getPoiDistanceFeatureName(name) ??
name; name;
const prev = merged.get(backendName); const prev = merged.get(backendName);

View file

@ -56,7 +56,7 @@ export const SMALLEST_VISIBLE_HEXAGON_RESOLUTION = Math.max(
// past the finest hexagon level so detail appears while still relatively // past the finest hexagon level so detail appears while still relatively
// zoomed out. (Each overlay additionally can't render below its own tile-data // zoomed out. (Each overlay additionally can't render below its own tile-data
// floor, OVERLAY_MIN_ZOOM, regardless of this limit.) // floor, OVERLAY_MIN_ZOOM, regardless of this limit.)
export const POSTCODE_ZOOM_THRESHOLD = 14; export const POSTCODE_ZOOM_THRESHOLD = 13.5;
export const POSTCODE_SEARCH_ZOOM = 16; export const POSTCODE_SEARCH_ZOOM = 16;
export const FEATURE_GRADIENT: { t: number; color: [number, number, number] }[] = [ export const FEATURE_GRADIENT: { t: number; color: [number, number, number] }[] = [
@ -264,7 +264,7 @@ export const STACKED_GROUPS: Record<
label: string; label: string;
/** If set, use this feature's stats for the total and info popup. Otherwise sum components. */ /** If set, use this feature's stats for the total and info popup. Otherwise sum components. */
feature?: string; feature?: string;
/** If set, display this feature's mean as the primary value (e.g. per-1k rate) instead of the absolute total. */ /** If set, display this feature's mean as the primary value (e.g. a percentage) instead of the absolute total. */
rateFeature?: string; rateFeature?: string;
/** Suffix shown after the total value (e.g. "avg/yr") */ /** Suffix shown after the total value (e.g. "avg/yr") */
unit?: string; unit?: string;
@ -275,30 +275,30 @@ export const STACKED_GROUPS: Record<
Crime: [ Crime: [
{ {
label: 'Serious crime', label: 'Serious crime',
feature: 'Serious crime (avg/yr)', feature: 'Serious crime (/yr, 7y)',
unit: '', unit: '',
components: [ components: [
'Violence and sexual offences (avg/yr)', 'Violence and sexual offences (/yr, 7y)',
'Robbery (avg/yr)', 'Robbery (/yr, 7y)',
'Burglary (avg/yr)', 'Burglary (/yr, 7y)',
'Possession of weapons (avg/yr)', 'Possession of weapons (/yr, 7y)',
], ],
}, },
{ {
label: 'Minor crime', label: 'Minor crime',
feature: 'Minor crime (avg/yr)', feature: 'Minor crime (/yr, 7y)',
unit: '', unit: '',
components: [ components: [
'Anti-social behaviour (avg/yr)', 'Anti-social behaviour (/yr, 7y)',
'Criminal damage and arson (avg/yr)', 'Criminal damage and arson (/yr, 7y)',
'Shoplifting (avg/yr)', 'Shoplifting (/yr, 7y)',
'Bicycle theft (avg/yr)', 'Bicycle theft (/yr, 7y)',
'Theft from the person (avg/yr)', 'Theft from the person (/yr, 7y)',
'Other theft (avg/yr)', 'Other theft (/yr, 7y)',
'Vehicle crime (avg/yr)', 'Vehicle crime (/yr, 7y)',
'Public order (avg/yr)', 'Public order (/yr, 7y)',
'Drugs (avg/yr)', 'Drugs (/yr, 7y)',
'Other crime (avg/yr)', 'Other crime (/yr, 7y)',
], ],
}, },
], ],
@ -493,20 +493,20 @@ export function getEnumValueColor(
/** Explicit colors for stacked bar segments. */ /** Explicit colors for stacked bar segments. */
export const STACKED_SEGMENT_COLORS: Record<string, string> = { export const STACKED_SEGMENT_COLORS: Record<string, string> = {
'Violence and sexual offences (avg/yr)': '#ef4444', 'Violence and sexual offences (/yr, 7y)': '#ef4444',
'Robbery (avg/yr)': '#f97316', 'Robbery (/yr, 7y)': '#f97316',
'Burglary (avg/yr)': '#eab308', 'Burglary (/yr, 7y)': '#eab308',
'Possession of weapons (avg/yr)': '#8b5cf6', 'Possession of weapons (/yr, 7y)': '#8b5cf6',
'Anti-social behaviour (avg/yr)': '#14b8a6', 'Anti-social behaviour (/yr, 7y)': '#14b8a6',
'Criminal damage and arson (avg/yr)': '#f97316', 'Criminal damage and arson (/yr, 7y)': '#f97316',
'Shoplifting (avg/yr)': '#ec4899', 'Shoplifting (/yr, 7y)': '#ec4899',
'Bicycle theft (avg/yr)': '#22c55e', 'Bicycle theft (/yr, 7y)': '#22c55e',
'Theft from the person (avg/yr)': '#d946ef', 'Theft from the person (/yr, 7y)': '#d946ef',
'Other theft (avg/yr)': '#06b6d4', 'Other theft (/yr, 7y)': '#06b6d4',
'Vehicle crime (avg/yr)': '#3b82f6', 'Vehicle crime (/yr, 7y)': '#3b82f6',
'Public order (avg/yr)': '#8b5cf6', 'Public order (/yr, 7y)': '#8b5cf6',
'Drugs (avg/yr)': '#22c55e', 'Drugs (/yr, 7y)': '#22c55e',
'Other crime (avg/yr)': '#6b7280', 'Other crime (/yr, 7y)': '#6b7280',
'% White': '#3b82f6', '% White': '#3b82f6',
'% South Asian': '#f97316', '% South Asian': '#f97316',
'% East Asian': '#eab308', '% East Asian': '#eab308',

View file

@ -0,0 +1,79 @@
import { describe, expect, it } from 'vitest';
import type { FeatureMeta } from '../types';
import {
SPECIFIC_CRIME_FEATURE_NAMES,
createSpecificCrimeFilterKey,
getDefaultSpecificCrimeFeatureName,
getSpecificCrimeFeatureName,
getSpecificCrimeFilterKeyId,
getSpecificCrimeType,
getSpecificCrimeWindow,
isSpecificCrimeFeatureName,
isSpecificCrimeFilterName,
normalizeSpecificCrimeFilters,
specificCrimeFeatureName,
withSpecificCrimeWindow,
} from './crime-filter';
const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
describe('specific-crime window helpers', () => {
it('recognizes both the 7-year and 2-year rate features', () => {
expect(isSpecificCrimeFeatureName('Burglary (/yr, 7y)')).toBe(true);
expect(isSpecificCrimeFeatureName('Burglary (/yr, 2y)')).toBe(true);
expect(isSpecificCrimeFeatureName('Burglary (avg/yr)')).toBe(false);
// The "Serious crime" / "Minor crime" aggregates stay separate sliders.
expect(isSpecificCrimeFeatureName('Serious crime (/yr, 7y)')).toBe(false);
});
it('extracts the window and bare type', () => {
expect(getSpecificCrimeWindow('Burglary (/yr, 2y)')).toBe('2y');
expect(getSpecificCrimeWindow('Burglary (/yr, 7y)')).toBe('7y');
expect(getSpecificCrimeWindow('Burglary')).toBeNull();
expect(getSpecificCrimeType('Violence and sexual offences (/yr, 2y)')).toBe(
'Violence and sexual offences'
);
expect(getSpecificCrimeType('not a crime')).toBeNull();
});
it('swaps a feature name between windows and round-trips', () => {
expect(withSpecificCrimeWindow('Burglary (/yr, 7y)', '2y')).toBe(
'Burglary (/yr, 2y)'
);
expect(withSpecificCrimeWindow('Burglary (/yr, 2y)', '7y')).toBe(
'Burglary (/yr, 7y)'
);
// Unrecognized names pass through untouched.
expect(withSpecificCrimeWindow('Burglary', '2y')).toBe('Burglary');
expect(specificCrimeFeatureName('Drugs', '2y')).toBe('Drugs (/yr, 2y)');
});
it('defaults to the 7-year window and enumerates 7-year dropdown names', () => {
const features = [numeric('Burglary (/yr, 7y)'), numeric('Burglary (/yr, 2y)')];
expect(getDefaultSpecificCrimeFeatureName(features)).toBe('Burglary (/yr, 7y)');
expect(SPECIFIC_CRIME_FEATURE_NAMES.every((n) => n.endsWith('(/yr, 7y)'))).toBe(true);
});
});
describe('specific-crime filter keys', () => {
it('resolves a 2-year filter key back to its backend column', () => {
const key = createSpecificCrimeFilterKey('Burglary (/yr, 2y)', 3);
expect(isSpecificCrimeFilterName(key)).toBe(true);
expect(getSpecificCrimeFilterKeyId(key)).toBe('3');
expect(getSpecificCrimeFeatureName(key)).toBe('Burglary (/yr, 2y)');
});
it('folds a bare 2-year crime feature into a multi-instance key', () => {
const normalized = normalizeSpecificCrimeFilters({
'Burglary (/yr, 2y)': [1, 5],
'Median price (£)': [0, 100],
});
const keys = Object.keys(normalized);
expect(keys.some((k) => isSpecificCrimeFilterName(k))).toBe(true);
const crimeKey = keys.find((k) => isSpecificCrimeFilterName(k))!;
expect(getSpecificCrimeFeatureName(crimeKey)).toBe('Burglary (/yr, 2y)');
expect(normalized[crimeKey]).toEqual([1, 5]);
// Non-crime filters are left untouched.
expect(normalized['Median price (£)']).toEqual([0, 100]);
});
});

View file

@ -1,26 +1,74 @@
import type { FeatureFilters, FeatureMeta } from '../types'; import type { FeatureFilters, FeatureMeta } from '../types';
import type { VariantFilterConfig } from './variant-filter';
export const SPECIFIC_CRIMES_FILTER_NAME = 'Specific crimes'; export const SPECIFIC_CRIMES_FILTER_NAME = 'Specific crimes';
export const SPECIFIC_CRIMES_FILTER_KEY_PREFIX = `${SPECIFIC_CRIMES_FILTER_NAME}:`; export const SPECIFIC_CRIMES_FILTER_KEY_PREFIX = `${SPECIFIC_CRIMES_FILTER_NAME}:`;
export const SPECIFIC_CRIME_FEATURE_NAMES = [ // The "pick one crime type" filter selects a single street-level category, with
'Violence and sexual offences (avg/yr)', // a toggle to measure it over the 7-year or the recent 2-year window. The 7-year
'Burglary (avg/yr)', // window is the default/primary one (also headlined in the area pane).
'Robbery (avg/yr)', export const SPECIFIC_CRIME_TYPES = [
'Vehicle crime (avg/yr)', 'Violence and sexual offences',
'Anti-social behaviour (avg/yr)', 'Burglary',
'Criminal damage and arson (avg/yr)', 'Robbery',
'Other theft (avg/yr)', 'Vehicle crime',
'Theft from the person (avg/yr)', 'Anti-social behaviour',
'Shoplifting (avg/yr)', 'Criminal damage and arson',
'Bicycle theft (avg/yr)', 'Other theft',
'Drugs (avg/yr)', 'Theft from the person',
'Possession of weapons (avg/yr)', 'Shoplifting',
'Public order (avg/yr)', 'Bicycle theft',
'Other crime (avg/yr)', 'Drugs',
'Possession of weapons',
'Public order',
'Other crime',
] as const; ] as const;
const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set<string>(SPECIFIC_CRIME_FEATURE_NAMES); export const SPECIFIC_CRIME_WINDOW_DEFAULT = '7y';
export const SPECIFIC_CRIME_WINDOWS = ['7y', '2y'] as const;
export type SpecificCrimeWindow = (typeof SPECIFIC_CRIME_WINDOWS)[number];
/** The backend rate-feature column for a crime type in a given window. */
export function specificCrimeFeatureName(type: string, window: SpecificCrimeWindow): string {
return `${type} (/yr, ${window})`;
}
// Canonical dropdown enumeration: one feature name per crime type, in the
// default (7-year) window. The window toggle re-points these to the chosen window.
export const SPECIFIC_CRIME_FEATURE_NAMES = SPECIFIC_CRIME_TYPES.map((type) =>
specificCrimeFeatureName(type, SPECIFIC_CRIME_WINDOW_DEFAULT)
);
// Every recognized specific-crime feature, across all windows. Used to detect /
// resolve a filter key's backend column for either window.
const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set<string>(
SPECIFIC_CRIME_TYPES.flatMap((type) =>
SPECIFIC_CRIME_WINDOWS.map((window) => specificCrimeFeatureName(type, window))
)
);
const SPECIFIC_CRIME_WINDOW_RE = / \(\/yr, (7y|2y)\)$/;
/** The window suffix of a specific-crime feature name (e.g. "2y"), or null. */
export function getSpecificCrimeWindow(featureName: string): SpecificCrimeWindow | null {
const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
return match ? (match[1] as SpecificCrimeWindow) : null;
}
/** The bare crime type of a specific-crime feature name (e.g. "Burglary"), or null. */
export function getSpecificCrimeType(featureName: string): string | null {
const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
return match ? featureName.slice(0, match.index) : null;
}
/** The same crime type's feature name in a different window (no-op if unrecognized). */
export function withSpecificCrimeWindow(
featureName: string,
window: SpecificCrimeWindow
): string {
const type = getSpecificCrimeType(featureName);
return type ? specificCrimeFeatureName(type, window) : featureName;
}
export function isSpecificCrimeFeatureName(name: string): boolean { export function isSpecificCrimeFeatureName(name: string): boolean {
return SPECIFIC_CRIME_FEATURE_NAME_SET.has(name); return SPECIFIC_CRIME_FEATURE_NAME_SET.has(name);
@ -101,7 +149,7 @@ export function getSpecificCrimeFilterMeta(features: FeatureMeta[]): FeatureMeta
description: description:
'Violence, burglary, robbery, drugs, shoplifting, vehicle crime, anti-social behaviour, public order, theft, and other crime types', 'Violence, burglary, robbery, drugs, shoplifting, vehicle crime, anti-social behaviour, public order, theft, and other crime types',
detail: detail:
'Filter by one street-level crime category at a time using an area-normalised crime density near each postcode (not a count of incidents per year).', 'Filter by one street-level crime category at a time, as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
source: 'crime', source: 'crime',
suffix: '', suffix: '',
}; };
@ -115,3 +163,26 @@ export function clampSpecificCrimeRange(
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]); const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))]; return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
} }
export const SPECIFIC_CRIME_VARIANT_CONFIG: VariantFilterConfig = {
filterName: SPECIFIC_CRIMES_FILTER_NAME,
featureNames: SPECIFIC_CRIME_FEATURE_NAMES,
dropdownLabelKey: 'filters.crimeType',
getFilterMeta: getSpecificCrimeFilterMeta,
getDefaultFeatureName: getDefaultSpecificCrimeFeatureName,
getFeatureName: getSpecificCrimeFeatureName,
replaceFilterKeySelection: replaceSpecificCrimeFilterKeySelection,
clampRange: clampSpecificCrimeRange,
// Dropdown shows the bare crime type; the window toggle carries the 7y/2y suffix.
getOptionLabelSource: (name) => getSpecificCrimeType(name) ?? name,
window: {
labelKey: 'filters.crimeWindow',
options: [
{ id: '7y', labelKey: 'filters.crimeWindow7y' },
{ id: '2y', labelKey: 'filters.crimeWindow2y' },
],
getWindow: getSpecificCrimeWindow,
withWindow: (name, windowId) =>
withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
},
};

View file

@ -0,0 +1,85 @@
import { describe, expect, it } from 'vitest';
import type { FeatureMeta } from '../types';
import {
CRIME_SEVERITY_FILTER_NAMES,
createCrimeSeverityFilterKey,
getCrimeSeverityFeatureName,
getCrimeSeverityFilterKeyId,
getCrimeSeverityFilterName,
getCrimeSeverityVariantConfig,
getDefaultCrimeSeverityFeatureName,
isCrimeSeverityFeatureName,
isCrimeSeverityFilterName,
normalizeCrimeSeverityFilters,
} from './crime-severity-filter';
const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
describe('crime-severity feature recognition', () => {
it('recognizes Serious/Minor crime in both windows only', () => {
expect(isCrimeSeverityFeatureName('Serious crime (/yr, 7y)')).toBe(true);
expect(isCrimeSeverityFeatureName('Serious crime (/yr, 2y)')).toBe(true);
expect(isCrimeSeverityFeatureName('Minor crime (/yr, 2y)')).toBe(true);
// Detailed categories belong to "Specific crimes", not severity.
expect(isCrimeSeverityFeatureName('Burglary (/yr, 7y)')).toBe(false);
// Bare names without a window suffix are not feature columns.
expect(isCrimeSeverityFeatureName('Serious crime')).toBe(false);
});
it('maps a feature or key back to its severity filter name', () => {
expect(getCrimeSeverityFilterName('Serious crime (/yr, 2y)')).toBe('Serious crime');
expect(getCrimeSeverityFilterName('Minor crime (/yr, 7y)')).toBe('Minor crime');
const key = createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0);
expect(getCrimeSeverityFilterName(key)).toBe('Serious crime');
expect(getCrimeSeverityFilterName('Burglary (/yr, 7y)')).toBeNull();
});
});
describe('crime-severity filter keys', () => {
it('resolves a 2-year key back to its backend column and id', () => {
const key = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 4);
expect(isCrimeSeverityFilterName(key)).toBe(true);
expect(getCrimeSeverityFilterKeyId(key)).toBe('4');
expect(getCrimeSeverityFeatureName(key)).toBe('Minor crime (/yr, 2y)');
});
it('defaults to the 7-year window when present', () => {
const features = [
numeric('Serious crime (/yr, 7y)'),
numeric('Serious crime (/yr, 2y)'),
];
expect(getDefaultCrimeSeverityFeatureName(features, 'Serious crime')).toBe(
'Serious crime (/yr, 7y)'
);
expect(getDefaultCrimeSeverityFeatureName(features, 'Minor crime')).toBeNull();
});
it('folds bare severity features but leaves specific crimes and others alone', () => {
const normalized = normalizeCrimeSeverityFilters({
'Serious crime (/yr, 2y)': [1, 5],
'Burglary (/yr, 7y)': [0, 3],
'Median price (£)': [0, 100],
});
const keys = Object.keys(normalized);
const severityKey = keys.find((k) => isCrimeSeverityFilterName(k))!;
expect(getCrimeSeverityFeatureName(severityKey)).toBe('Serious crime (/yr, 2y)');
expect(normalized[severityKey]).toEqual([1, 5]);
// Specific-crime + plain features pass through untouched.
expect(normalized['Burglary (/yr, 7y)']).toEqual([0, 3]);
expect(normalized['Median price (£)']).toEqual([0, 100]);
});
});
describe('crime-severity variant config', () => {
it('exposes a single variant per severity with a 7y/2y window toggle', () => {
for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
const config = getCrimeSeverityVariantConfig(filterName);
expect(config.filterName).toBe(filterName);
expect(config.featureNames).toEqual([`${filterName} (/yr, 7y)`]);
expect(config.window?.options.map((o) => o.id)).toEqual(['7y', '2y']);
// Switching window re-points to the same severity, other window.
const sevenYear = `${filterName} (/yr, 7y)`;
expect(config.window?.withWindow(sevenYear, '2y')).toBe(`${filterName} (/yr, 2y)`);
}
});
});

View file

@ -0,0 +1,193 @@
import type { FeatureFilters, FeatureMeta } from '../types';
import type { VariantFilterConfig } from './variant-filter';
import {
SPECIFIC_CRIME_WINDOW_DEFAULT,
clampSpecificCrimeRange,
getSpecificCrimeType,
getSpecificCrimeWindow,
specificCrimeFeatureName,
withSpecificCrimeWindow,
type SpecificCrimeWindow,
} from './crime-filter';
// "Serious crime" and "Minor crime" are aggregate severity rollups (they overlap
// the 14 detailed categories, so they must stay OUT of the "Specific crimes"
// dropdown). Each is a single feature with a 7-year/2-year window toggle — the
// same folding the specific-crimes card has, but with no variant dropdown (one
// variant). Modeled on the multi-filter-name POI pattern: one lib, two filter
// names, distinguished by their key prefix / bare crime type.
export const SERIOUS_CRIME_FILTER_NAME = 'Serious crime';
export const MINOR_CRIME_FILTER_NAME = 'Minor crime';
export const CRIME_SEVERITY_FILTER_NAMES = [
SERIOUS_CRIME_FILTER_NAME,
MINOR_CRIME_FILTER_NAME,
] as const;
export type CrimeSeverityFilterName = (typeof CRIME_SEVERITY_FILTER_NAMES)[number];
const CRIME_SEVERITY_TYPE_SET = new Set<string>(CRIME_SEVERITY_FILTER_NAMES);
const CRIME_SEVERITY_DETAILS: Record<CrimeSeverityFilterName, { description: string; detail: string }> = {
[SERIOUS_CRIME_FILTER_NAME]: {
description: 'Violence, robbery, burglary and weapons possession near the postcode',
detail:
'Combined count of the more serious street-level categories (violence and sexual offences, robbery, burglary, possession of weapons), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
},
[MINOR_CRIME_FILTER_NAME]: {
description: 'Anti-social behaviour, theft, criminal damage, drugs and public order near the postcode',
detail:
'Combined count of the lower-severity street-level categories (anti-social behaviour, theft, criminal damage and arson, drugs, public order), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
},
};
function keyPrefix(filterName: CrimeSeverityFilterName): string {
return `${filterName}:`;
}
/** The severity filter a name belongs to (from its key prefix or bare feature name), or null. */
export function getCrimeSeverityFilterName(name: string): CrimeSeverityFilterName | null {
for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
if (name.startsWith(keyPrefix(filterName))) return filterName;
}
const type = getSpecificCrimeType(name);
return type && CRIME_SEVERITY_TYPE_SET.has(type) ? (type as CrimeSeverityFilterName) : null;
}
export function isCrimeSeverityFeatureName(name: string): boolean {
const type = getSpecificCrimeType(name);
return type != null && CRIME_SEVERITY_TYPE_SET.has(type);
}
export function isCrimeSeverityFilterName(name: string): boolean {
return getCrimeSeverityFilterName(name) != null;
}
export function createCrimeSeverityFilterKey(
filterName: CrimeSeverityFilterName,
featureName: string,
id: number | string
): string {
return `${keyPrefix(filterName)}${encodeURIComponent(featureName)}:${id}`;
}
export function getCrimeSeverityFilterKeyId(name: string): string | null {
const filterName = getCrimeSeverityFilterName(name);
if (!filterName) return null;
const prefix = keyPrefix(filterName);
if (!name.startsWith(prefix)) return null; // a bare feature name has no id
const rest = name.substring(prefix.length);
const lastColon = rest.lastIndexOf(':');
return lastColon === -1 ? null : rest.substring(lastColon + 1);
}
export function parseCrimeSeverityFilterKey(name: string): string | null {
const filterName = getCrimeSeverityFilterName(name);
if (!filterName) return null;
const prefix = keyPrefix(filterName);
if (!name.startsWith(prefix)) return null;
const rest = name.substring(prefix.length);
const lastColon = rest.lastIndexOf(':');
if (lastColon === -1) return null;
const decoded = decodeURIComponent(rest.substring(0, lastColon));
return isCrimeSeverityFeatureName(decoded) ? decoded : null;
}
export function getCrimeSeverityFeatureName(name: string): string | null {
if (isCrimeSeverityFeatureName(name)) return name;
return parseCrimeSeverityFilterKey(name);
}
export function replaceCrimeSeverityFilterKeySelection(key: string, featureName: string): string {
const filterName = getCrimeSeverityFilterName(key) ?? getCrimeSeverityFilterName(featureName);
const id = getCrimeSeverityFilterKeyId(key) ?? '0';
// filterName is always resolvable here: the key carries a prefix and a window
// switch keeps the same severity type. Fall back defensively to the key as-is.
if (!filterName) return key;
return createCrimeSeverityFilterKey(filterName, featureName, id);
}
/** The default (7-year) backend feature for a severity, if present in `features`. */
export function getDefaultCrimeSeverityFeatureName(
features: FeatureMeta[],
filterName: CrimeSeverityFilterName
): string | null {
const name = specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT);
return features.some((feature) => feature.name === name) ? name : null;
}
export function normalizeCrimeSeverityFilters(filters: FeatureFilters): FeatureFilters {
let changed = false;
const next: FeatureFilters = {};
for (const [name, value] of Object.entries(filters)) {
if (isCrimeSeverityFeatureName(name)) {
const filterName = getCrimeSeverityFilterName(name)!;
next[createCrimeSeverityFilterKey(filterName, name, Object.keys(next).length)] = value;
changed = true;
continue;
}
next[name] = value;
}
return changed ? next : filters;
}
export function getCrimeSeverityFilterMeta(
features: FeatureMeta[],
filterName: CrimeSeverityFilterName
): FeatureMeta {
const sourceFeatureName = getDefaultCrimeSeverityFeatureName(features, filterName);
const sourceFeature = sourceFeatureName
? features.find((feature) => feature.name === sourceFeatureName)
: undefined;
return {
name: filterName,
type: 'numeric',
group: 'Crime',
min: sourceFeature?.min ?? 0,
max: sourceFeature?.max ?? 100,
step: 1,
description: CRIME_SEVERITY_DETAILS[filterName].description,
detail: CRIME_SEVERITY_DETAILS[filterName].detail,
source: 'crime',
suffix: '',
};
}
export function clampCrimeSeverityRange(
value: [number, number],
feature?: FeatureMeta
): [number, number] {
return clampSpecificCrimeRange(value, feature);
}
/** The variant-filter config for one severity (single variant + 7y/2y window toggle). */
export function getCrimeSeverityVariantConfig(
filterName: CrimeSeverityFilterName
): VariantFilterConfig {
return {
filterName,
// One variant: the severity itself, in the default window. The card hides the
// (single-option) dropdown and exposes only the window toggle.
featureNames: [specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT)],
dropdownLabelKey: 'filters.crimeType',
getFilterMeta: (features) => getCrimeSeverityFilterMeta(features, filterName),
getDefaultFeatureName: (features) => getDefaultCrimeSeverityFeatureName(features, filterName),
getFeatureName: getCrimeSeverityFeatureName,
replaceFilterKeySelection: replaceCrimeSeverityFilterKeySelection,
clampRange: clampCrimeSeverityRange,
window: {
labelKey: 'filters.crimeWindow',
options: [
{ id: '7y', labelKey: 'filters.crimeWindow7y' },
{ id: '2y', labelKey: 'filters.crimeWindow2y' },
],
getWindow: getSpecificCrimeWindow,
withWindow: (name, windowId) =>
withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
},
};
}

View file

@ -6,24 +6,47 @@
export interface CrimeTypeDef { export interface CrimeTypeDef {
value: string; value: string;
/** English label (fallback / reference). */
label: string; label: string;
/** i18n key under `crimeTypes.*` used to render the selector label. */
labelKey: string;
} }
export const CRIME_TYPES: readonly CrimeTypeDef[] = [ export const CRIME_TYPES: readonly CrimeTypeDef[] = [
{ value: 'Violence and sexual offences', label: 'Violence & sexual offences' }, {
{ value: 'Anti-social behaviour', label: 'Anti-social behaviour' }, value: 'Violence and sexual offences',
{ value: 'Criminal damage and arson', label: 'Criminal damage & arson' }, label: 'Violence & sexual offences',
{ value: 'Public order', label: 'Public order' }, labelKey: 'crimeTypes.violenceAndSexualOffences',
{ value: 'Shoplifting', label: 'Shoplifting' }, },
{ value: 'Vehicle crime', label: 'Vehicle crime' }, {
{ value: 'Burglary', label: 'Burglary' }, value: 'Anti-social behaviour',
{ value: 'Other theft', label: 'Other theft' }, label: 'Anti-social behaviour',
{ value: 'Theft from the person', label: 'Theft from the person' }, labelKey: 'crimeTypes.antiSocialBehaviour',
{ value: 'Bicycle theft', label: 'Bicycle theft' }, },
{ value: 'Drugs', label: 'Drugs' }, {
{ value: 'Robbery', label: 'Robbery' }, value: 'Criminal damage and arson',
{ value: 'Possession of weapons', label: 'Possession of weapons' }, label: 'Criminal damage & arson',
{ value: 'Other crime', label: 'Other crime' }, labelKey: 'crimeTypes.criminalDamageAndArson',
},
{ value: 'Public order', label: 'Public order', labelKey: 'crimeTypes.publicOrder' },
{ value: 'Shoplifting', label: 'Shoplifting', labelKey: 'crimeTypes.shoplifting' },
{ value: 'Vehicle crime', label: 'Vehicle crime', labelKey: 'crimeTypes.vehicleCrime' },
{ value: 'Burglary', label: 'Burglary', labelKey: 'crimeTypes.burglary' },
{ value: 'Other theft', label: 'Other theft', labelKey: 'crimeTypes.otherTheft' },
{
value: 'Theft from the person',
label: 'Theft from the person',
labelKey: 'crimeTypes.theftFromThePerson',
},
{ value: 'Bicycle theft', label: 'Bicycle theft', labelKey: 'crimeTypes.bicycleTheft' },
{ value: 'Drugs', label: 'Drugs', labelKey: 'crimeTypes.drugs' },
{ value: 'Robbery', label: 'Robbery', labelKey: 'crimeTypes.robbery' },
{
value: 'Possession of weapons',
label: 'Possession of weapons',
labelKey: 'crimeTypes.possessionOfWeapons',
},
{ value: 'Other crime', label: 'Other crime', labelKey: 'crimeTypes.otherCrime' },
] as const; ] as const;
export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value); export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value);

View file

@ -404,7 +404,12 @@ const FEATURE_ICON_PATHS: Record<string, ReactNode> = {
* Returns a complete SVG icon element for a given feature name, or null if unmapped. * Returns a complete SVG icon element for a given feature name, or null if unmapped.
*/ */
export function getFeatureIcon(featureName: string, className: string): ReactElement | null { export function getFeatureIcon(featureName: string, className: string): ReactElement | null {
const paths = FEATURE_ICON_PATHS[featureName]; // Crime features ("Burglary (/yr, 7y)") share the bare type's legacy
// "(avg/yr)" icon key; both windows use the same icon.
const rateMatch = featureName.match(/^(.*) \(\/yr, \d+y\)$/);
const paths =
FEATURE_ICON_PATHS[featureName] ??
(rateMatch ? FEATURE_ICON_PATHS[`${rateMatch[1]} (avg/yr)`] : undefined);
if (!paths) return null; if (!paths) return null;
return ( return (
<svg <svg

View file

@ -1,9 +1,16 @@
import type { FeatureFilters, FeatureMeta } from '../types';
import type { VariantFilterConfig } from './variant-filter';
export const QUALIFICATIONS_FILTER_NAME = 'Qualifications';
export const QUALIFICATIONS_FILTER_KEY_PREFIX = `${QUALIFICATIONS_FILTER_NAME}:`;
/** /**
* The Census 2021 qualification-breakdown features (TS067). These render as a * The Census 2021 "highest level of qualification" bands (TS067). They sum to
* single stacked "Qualifications" composition in the area pane (see * 100% per neighbourhood and render as a single stacked "Qualifications"
* STACKED_GROUPS["Neighbours"] in consts.ts) and are display-only: they are * composition in the area pane (see STACKED_GROUPS["Neighbours"] in consts.ts).
* hidden from the filter browser rather than offered as seven individual * In the filter browser they fold into one "Qualifications" filter whose
* sliders, so the breakdown reads as a ratio without cluttering the filter list. * dropdown selects a band including "% Degree or higher" rather than seven
* separate sliders.
*/ */
export const QUALIFICATION_FEATURE_NAMES = [ export const QUALIFICATION_FEATURE_NAMES = [
'% No qualifications', '% No qualifications',
@ -20,3 +27,103 @@ const QUALIFICATION_FEATURE_NAME_SET = new Set<string>(QUALIFICATION_FEATURE_NAM
export function isQualificationFeatureName(name: string): boolean { export function isQualificationFeatureName(name: string): boolean {
return QUALIFICATION_FEATURE_NAME_SET.has(name); return QUALIFICATION_FEATURE_NAME_SET.has(name);
} }
export function isQualificationFilterName(name: string): boolean {
return isQualificationFeatureName(name) || name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX);
}
export function createQualificationFilterKey(featureName: string, id: number | string): string {
return `${QUALIFICATIONS_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
}
export function getQualificationFilterKeyId(name: string): string | null {
if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
const lastColon = rest.lastIndexOf(':');
return lastColon === -1 ? null : rest.substring(lastColon + 1);
}
export function parseQualificationFilterKey(name: string): string | null {
if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
const lastColon = rest.lastIndexOf(':');
if (lastColon === -1) return null;
const decoded = decodeURIComponent(rest.substring(0, lastColon));
return isQualificationFeatureName(decoded) ? decoded : null;
}
export function getQualificationFeatureName(name: string): string | null {
if (isQualificationFeatureName(name)) return name;
return parseQualificationFilterKey(name);
}
export function replaceQualificationFilterKeySelection(key: string, featureName: string): string {
const id = getQualificationFilterKeyId(key) ?? '0';
return createQualificationFilterKey(featureName, id);
}
export function getDefaultQualificationFeatureName(features: FeatureMeta[]): string | null {
return (
QUALIFICATION_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ??
null
);
}
export function normalizeQualificationFilters(filters: FeatureFilters): FeatureFilters {
let changed = false;
const next: FeatureFilters = {};
for (const [name, value] of Object.entries(filters)) {
if (isQualificationFeatureName(name)) {
next[createQualificationFilterKey(name, Object.keys(next).length)] = value;
changed = true;
continue;
}
next[name] = value;
}
return changed ? next : filters;
}
export function getQualificationFilterMeta(features: FeatureMeta[]): FeatureMeta {
const sourceFeatureName = getDefaultQualificationFeatureName(features);
const sourceFeature = sourceFeatureName
? features.find((feature) => feature.name === sourceFeatureName)
: undefined;
return {
name: QUALIFICATIONS_FILTER_NAME,
type: 'numeric',
group: 'Neighbours',
min: sourceFeature?.min ?? 0,
max: sourceFeature?.max ?? 100,
step: 1,
description:
'Share of residents (16+) by highest qualification, from no qualifications to a degree or higher',
detail:
'Filter by one Census 2021 (TS067) highest-qualification band at a time, e.g. the percentage of residents whose highest qualification is a degree or higher.',
source: 'census-2021',
suffix: '%',
};
}
export function clampQualificationRange(
value: [number, number],
feature?: FeatureMeta
): [number, number] {
const min = feature?.histogram?.min ?? feature?.min ?? 0;
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
}
export const QUALIFICATION_VARIANT_CONFIG: VariantFilterConfig = {
filterName: QUALIFICATIONS_FILTER_NAME,
featureNames: QUALIFICATION_FEATURE_NAMES,
dropdownLabelKey: 'filters.qualificationLevel',
getFilterMeta: getQualificationFilterMeta,
getDefaultFeatureName: getDefaultQualificationFeatureName,
getFeatureName: getQualificationFeatureName,
replaceFilterKeySelection: replaceQualificationFilterKeySelection,
clampRange: clampQualificationRange,
};

View file

@ -0,0 +1,121 @@
import type { FeatureFilters, FeatureMeta } from '../types';
import type { VariantFilterConfig } from './variant-filter';
export const TENURE_FILTER_NAME = 'Tenure';
export const TENURE_FILTER_KEY_PREFIX = `${TENURE_FILTER_NAME}:`;
/**
* The Census 2021 housing tenure categories (TS054). They sum to 100% per
* neighbourhood and render as a single stacked "Tenure" composition in the area
* pane (see STACKED_GROUPS["Neighbours"] in consts.ts). In the filter browser
* they fold into one "Tenure" filter whose dropdown selects a category
* owner-occupied, social rent or private rent rather than three separate
* sliders.
*/
export const TENURE_FEATURE_NAMES = [
'% Owner occupied',
'% Social rent',
'% Private rent',
] as const;
const TENURE_FEATURE_NAME_SET = new Set<string>(TENURE_FEATURE_NAMES);
export function isTenureFeatureName(name: string): boolean {
return TENURE_FEATURE_NAME_SET.has(name);
}
export function isTenureFilterName(name: string): boolean {
return isTenureFeatureName(name) || name.startsWith(TENURE_FILTER_KEY_PREFIX);
}
export function createTenureFilterKey(featureName: string, id: number | string): string {
return `${TENURE_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
}
export function getTenureFilterKeyId(name: string): string | null {
if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
const lastColon = rest.lastIndexOf(':');
return lastColon === -1 ? null : rest.substring(lastColon + 1);
}
export function parseTenureFilterKey(name: string): string | null {
if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
const lastColon = rest.lastIndexOf(':');
if (lastColon === -1) return null;
const decoded = decodeURIComponent(rest.substring(0, lastColon));
return isTenureFeatureName(decoded) ? decoded : null;
}
export function getTenureFeatureName(name: string): string | null {
if (isTenureFeatureName(name)) return name;
return parseTenureFilterKey(name);
}
export function replaceTenureFilterKeySelection(key: string, featureName: string): string {
const id = getTenureFilterKeyId(key) ?? '0';
return createTenureFilterKey(featureName, id);
}
export function getDefaultTenureFeatureName(features: FeatureMeta[]): string | null {
return (
TENURE_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ?? null
);
}
export function normalizeTenureFilters(filters: FeatureFilters): FeatureFilters {
let changed = false;
const next: FeatureFilters = {};
for (const [name, value] of Object.entries(filters)) {
if (isTenureFeatureName(name)) {
next[createTenureFilterKey(name, Object.keys(next).length)] = value;
changed = true;
continue;
}
next[name] = value;
}
return changed ? next : filters;
}
export function getTenureFilterMeta(features: FeatureMeta[]): FeatureMeta {
const sourceFeatureName = getDefaultTenureFeatureName(features);
const sourceFeature = sourceFeatureName
? features.find((feature) => feature.name === sourceFeatureName)
: undefined;
return {
name: TENURE_FILTER_NAME,
type: 'numeric',
group: 'Neighbours',
min: sourceFeature?.min ?? 0,
max: sourceFeature?.max ?? 100,
step: 1,
description:
'Share of households that own their home, rent from a social landlord, or rent privately',
detail:
'Filter by one Census 2021 (TS054) housing tenure category at a time, e.g. the percentage of households that own their home.',
source: 'census-2021',
suffix: '%',
};
}
export function clampTenureRange(value: [number, number], feature?: FeatureMeta): [number, number] {
const min = feature?.histogram?.min ?? feature?.min ?? 0;
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
}
export const TENURE_VARIANT_CONFIG: VariantFilterConfig = {
filterName: TENURE_FILTER_NAME,
featureNames: TENURE_FEATURE_NAMES,
dropdownLabelKey: 'filters.tenureType',
getFilterMeta: getTenureFilterMeta,
getDefaultFeatureName: getDefaultTenureFeatureName,
getFeatureName: getTenureFeatureName,
replaceFilterKeySelection: replaceTenureFilterKeySelection,
clampRange: clampTenureRange,
};

View file

@ -5,8 +5,11 @@ import { parseUrlState, stateToParams } from './url-state';
import { INITIAL_VIEW_STATE } from './consts'; import { INITIAL_VIEW_STATE } from './consts';
import { createSchoolFilterKey } from './school-filter'; import { createSchoolFilterKey } from './school-filter';
import { createSpecificCrimeFilterKey } from './crime-filter'; import { createSpecificCrimeFilterKey } from './crime-filter';
import { createCrimeSeverityFilterKey } from './crime-severity-filter';
import { createElectionVoteShareFilterKey } from './election-filter'; import { createElectionVoteShareFilterKey } from './election-filter';
import { createEthnicityFilterKey } from './ethnicity-filter'; import { createEthnicityFilterKey } from './ethnicity-filter';
import { createQualificationFilterKey } from './qualification-filter';
import { createTenureFilterKey } from './tenure-filter';
import { import {
POI_COUNT_2KM_FILTER_NAME, POI_COUNT_2KM_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME, TRANSPORT_DISTANCE_FILTER_NAME,
@ -405,8 +408,8 @@ describe('url-state', () => {
}); });
it('round-trips repeated specific crime filters with dedicated URL params', () => { it('round-trips repeated specific crime filters with dedicated URL params', () => {
const burglary = createSpecificCrimeFilterKey('Burglary (avg/yr)', 1); const burglary = createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1);
const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2); const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2);
const params = stateToParams( const params = stateToParams(
null, null,
@ -420,8 +423,8 @@ describe('url-state', () => {
); );
expect(params.getAll('crime')).toEqual([ expect(params.getAll('crime')).toEqual([
'Burglary (avg/yr):0:5', 'Burglary (/yr, 7y):0:5',
'Vehicle crime (avg/yr):1:10', 'Vehicle crime (/yr, 7y):1:10',
]); ]);
expect(params.getAll('filter')).toEqual([]); expect(params.getAll('filter')).toEqual([]);
@ -429,8 +432,42 @@ describe('url-state', () => {
const state = parseUrlState(); const state = parseUrlState();
expect(state.filters).toEqual({ expect(state.filters).toEqual({
[createSpecificCrimeFilterKey('Burglary (avg/yr)', 0)]: [0, 5], [createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 0)]: [0, 5],
[createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 1)]: [1, 10], [createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 1)]: [1, 10],
});
});
it('round-trips serious/minor crime severity filters with dedicated URL params', () => {
const serious = createCrimeSeverityFilterKey(
'Serious crime',
'Serious crime (/yr, 7y)',
0
);
const minor = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1);
const params = stateToParams(
null,
{
[serious]: [0, 12],
[minor]: [3, 40],
},
[],
new Set(),
'area'
);
expect(params.getAll('crimeSeverity')).toEqual([
'Serious crime (/yr, 7y):0:12',
'Minor crime (/yr, 2y):3:40',
]);
expect(params.getAll('filter')).toEqual([]);
window.history.replaceState({}, '', `/?${params.toString()}`);
const state = parseUrlState();
expect(state.filters).toEqual({
[createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0)]: [0, 12],
[createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1)]: [3, 40],
}); });
}); });
@ -488,6 +525,65 @@ describe('url-state', () => {
}); });
}); });
it('round-trips repeated qualification filters with dedicated URL params', () => {
// "% Degree or higher" exercises the leading-`%` + space encoding path.
const degree = createQualificationFilterKey('% Degree or higher', 1);
const noQuals = createQualificationFilterKey('% No qualifications', 2);
const params = stateToParams(
null,
{
[degree]: [20, 60],
[noQuals]: [0, 25],
},
[],
new Set(),
'area'
);
expect(params.getAll('qualification')).toEqual([
'% Degree or higher:20:60',
'% No qualifications:0:25',
]);
expect(params.getAll('filter')).toEqual([]);
window.history.replaceState({}, '', `/?${params.toString()}`);
const state = parseUrlState();
expect(state.filters).toEqual({
[createQualificationFilterKey('% Degree or higher', 0)]: [20, 60],
[createQualificationFilterKey('% No qualifications', 1)]: [0, 25],
});
});
it('round-trips repeated tenure filters with dedicated URL params', () => {
// "% Owner occupied" exercises the leading-`%` + space encoding path.
const owner = createTenureFilterKey('% Owner occupied', 1);
const privateRent = createTenureFilterKey('% Private rent', 2);
const params = stateToParams(
null,
{
[owner]: [20, 60],
[privateRent]: [0, 25],
},
[],
new Set(),
'area'
);
expect(params.getAll('tenure')).toEqual(['% Owner occupied:20:60', '% Private rent:0:25']);
expect(params.getAll('filter')).toEqual([]);
window.history.replaceState({}, '', `/?${params.toString()}`);
const state = parseUrlState();
expect(state.filters).toEqual({
[createTenureFilterKey('% Owner occupied', 0)]: [20, 60],
[createTenureFilterKey('% Private rent', 1)]: [0, 25],
});
});
it('round-trips repeated amenity distance filters with dedicated URL params', () => { it('round-trips repeated amenity distance filters with dedicated URL params', () => {
const park = createPoiDistanceFilterKey('Distance to nearest amenity (Park) (km)', 3); const park = createPoiDistanceFilterKey('Distance to nearest amenity (Park) (km)', 3);
const cafe = createPoiDistanceFilterKey('Distance to nearest amenity (Café) (km)', 4); const cafe = createPoiDistanceFilterKey('Distance to nearest amenity (Café) (km)', 4);

View file

@ -22,6 +22,13 @@ import {
isSpecificCrimeFeatureName, isSpecificCrimeFeatureName,
isSpecificCrimeFilterName, isSpecificCrimeFilterName,
} from './crime-filter'; } from './crime-filter';
import {
createCrimeSeverityFilterKey,
getCrimeSeverityFeatureName,
getCrimeSeverityFilterName,
isCrimeSeverityFeatureName,
isCrimeSeverityFilterName,
} from './crime-severity-filter';
import { import {
ELECTION_VOTE_SHARE_FILTER_NAME, ELECTION_VOTE_SHARE_FILTER_NAME,
createElectionVoteShareFilterKey, createElectionVoteShareFilterKey,
@ -36,6 +43,20 @@ import {
isEthnicityFeatureName, isEthnicityFeatureName,
isEthnicityFilterName, isEthnicityFilterName,
} from './ethnicity-filter'; } from './ethnicity-filter';
import {
QUALIFICATIONS_FILTER_NAME,
createQualificationFilterKey,
getQualificationFeatureName,
isQualificationFeatureName,
isQualificationFilterName,
} from './qualification-filter';
import {
TENURE_FILTER_NAME,
createTenureFilterKey,
getTenureFeatureName,
isTenureFeatureName,
isTenureFilterName,
} from './tenure-filter';
import { import {
POI_DISTANCE_FILTER_NAME, POI_DISTANCE_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME, TRANSPORT_DISTANCE_FILTER_NAME,
@ -83,8 +104,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
const filterParams = params.getAll('filter'); const filterParams = params.getAll('filter');
const schoolParams = params.getAll('school'); const schoolParams = params.getAll('school');
const crimeParams = params.getAll('crime'); const crimeParams = params.getAll('crime');
const crimeSeverityParams = params.getAll('crimeSeverity');
const voteShareParams = params.getAll('voteShare'); const voteShareParams = params.getAll('voteShare');
const ethnicityParams = params.getAll('ethnicity'); const ethnicityParams = params.getAll('ethnicity');
const qualificationParams = params.getAll('qualification');
const tenureParams = params.getAll('tenure');
const amenityDistanceParams = params.getAll('amenityDistance'); const amenityDistanceParams = params.getAll('amenityDistance');
const transportDistanceParams = params.getAll('transportDistance'); const transportDistanceParams = params.getAll('transportDistance');
const amenityCount2KmParams = params.getAll('amenityCount2km'); const amenityCount2KmParams = params.getAll('amenityCount2km');
@ -93,8 +117,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filterParams.length === 0 && filterParams.length === 0 &&
schoolParams.length === 0 && schoolParams.length === 0 &&
crimeParams.length === 0 && crimeParams.length === 0 &&
crimeSeverityParams.length === 0 &&
voteShareParams.length === 0 && voteShareParams.length === 0 &&
ethnicityParams.length === 0 && ethnicityParams.length === 0 &&
qualificationParams.length === 0 &&
tenureParams.length === 0 &&
amenityDistanceParams.length === 0 && amenityDistanceParams.length === 0 &&
transportDistanceParams.length === 0 && transportDistanceParams.length === 0 &&
amenityCount2KmParams.length === 0 && amenityCount2KmParams.length === 0 &&
@ -155,6 +182,19 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filters[createSpecificCrimeFilterKey(featureName, index)] = [min, max]; filters[createSpecificCrimeFilterKey(featureName, index)] = [min, max];
}); });
crimeSeverityParams.forEach((entry, index) => {
const parts = entry.split(':');
if (parts.length < 3) return;
const featureName = parts.slice(0, -2).join(':');
const min = Number(parts[parts.length - 2]);
const max = Number(parts[parts.length - 1]);
const filterName = getCrimeSeverityFilterName(featureName);
if (!isCrimeSeverityFeatureName(featureName) || !filterName || isNaN(min) || isNaN(max)) {
return;
}
filters[createCrimeSeverityFilterKey(filterName, featureName, index)] = [min, max];
});
voteShareParams.forEach((entry, index) => { voteShareParams.forEach((entry, index) => {
const parts = entry.split(':'); const parts = entry.split(':');
if (parts.length < 3) return; if (parts.length < 3) return;
@ -179,6 +219,30 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filters[createEthnicityFilterKey(featureName, index)] = [min, max]; filters[createEthnicityFilterKey(featureName, index)] = [min, max];
}); });
qualificationParams.forEach((entry, index) => {
const parts = entry.split(':');
if (parts.length < 3) return;
const featureName = parts.slice(0, -2).join(':');
const min = Number(parts[parts.length - 2]);
const max = Number(parts[parts.length - 1]);
if (!isQualificationFeatureName(featureName) || isNaN(min) || isNaN(max)) {
return;
}
filters[createQualificationFilterKey(featureName, index)] = [min, max];
});
tenureParams.forEach((entry, index) => {
const parts = entry.split(':');
if (parts.length < 3) return;
const featureName = parts.slice(0, -2).join(':');
const min = Number(parts[parts.length - 2]);
const max = Number(parts[parts.length - 1]);
if (!isTenureFeatureName(featureName) || isNaN(min) || isNaN(max)) {
return;
}
filters[createTenureFilterKey(featureName, index)] = [min, max];
});
const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => { const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => {
entries.forEach((entry, index) => { entries.forEach((entry, index) => {
const parts = entry.split(':'); const parts = entry.split(':');
@ -401,6 +465,13 @@ export function stateToParams(
continue; continue;
} }
const crimeSeverityFeatureName = getCrimeSeverityFeatureName(name);
if (crimeSeverityFeatureName && isCrimeSeverityFilterName(name)) {
const [min, max] = value as [number, number];
params.append('crimeSeverity', `${crimeSeverityFeatureName}:${min}:${max}`);
continue;
}
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name); const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
if (electionVoteShareFeatureName && isElectionVoteShareFilterName(name)) { if (electionVoteShareFeatureName && isElectionVoteShareFilterName(name)) {
const [min, max] = value as [number, number]; const [min, max] = value as [number, number];
@ -415,6 +486,20 @@ export function stateToParams(
continue; continue;
} }
const qualificationFeatureName = getQualificationFeatureName(name);
if (qualificationFeatureName && isQualificationFilterName(name)) {
const [min, max] = value as [number, number];
params.append('qualification', `${qualificationFeatureName}:${min}:${max}`);
continue;
}
const tenureFeatureName = getTenureFeatureName(name);
if (tenureFeatureName && isTenureFilterName(name)) {
const [min, max] = value as [number, number];
params.append('tenure', `${tenureFeatureName}:${min}:${max}`);
continue;
}
const amenityDistanceFeatureName = getPoiDistanceFeatureName(name); const amenityDistanceFeatureName = getPoiDistanceFeatureName(name);
if (amenityDistanceFeatureName && isPoiDistanceFilterName(name)) { if (amenityDistanceFeatureName && isPoiDistanceFilterName(name)) {
const [min, max] = value as [number, number]; const [min, max] = value as [number, number];
@ -522,8 +607,11 @@ export function summarizeParams(queryString: string): string {
const filterParams = params.getAll('filter'); const filterParams = params.getAll('filter');
const schoolParams = params.getAll('school'); const schoolParams = params.getAll('school');
const crimeParams = params.getAll('crime'); const crimeParams = params.getAll('crime');
const crimeSeverityParams = params.getAll('crimeSeverity');
const voteShareParams = params.getAll('voteShare'); const voteShareParams = params.getAll('voteShare');
const ethnicityParams = params.getAll('ethnicity'); const ethnicityParams = params.getAll('ethnicity');
const qualificationParams = params.getAll('qualification');
const tenureParams = params.getAll('tenure');
const amenityDistanceParams = params.getAll('amenityDistance'); const amenityDistanceParams = params.getAll('amenityDistance');
const transportDistanceParams = params.getAll('transportDistance'); const transportDistanceParams = params.getAll('transportDistance');
const amenityCount2KmParams = params.getAll('amenityCount2km'); const amenityCount2KmParams = params.getAll('amenityCount2km');
@ -532,8 +620,11 @@ export function summarizeParams(queryString: string): string {
filterParams.length > 0 || filterParams.length > 0 ||
schoolParams.length > 0 || schoolParams.length > 0 ||
crimeParams.length > 0 || crimeParams.length > 0 ||
crimeSeverityParams.length > 0 ||
voteShareParams.length > 0 || voteShareParams.length > 0 ||
ethnicityParams.length > 0 || ethnicityParams.length > 0 ||
qualificationParams.length > 0 ||
tenureParams.length > 0 ||
amenityDistanceParams.length > 0 || amenityDistanceParams.length > 0 ||
transportDistanceParams.length > 0 || transportDistanceParams.length > 0 ||
amenityCount2KmParams.length > 0 || amenityCount2KmParams.length > 0 ||
@ -544,8 +635,11 @@ export function summarizeParams(queryString: string): string {
const colonIdx = entry.indexOf(':'); const colonIdx = entry.indexOf(':');
const name = colonIdx > 0 ? entry.substring(0, colonIdx) : entry; const name = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME; if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
if (isCrimeSeverityFeatureName(name)) return getCrimeSeverityFilterName(name) ?? name;
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME; if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME; if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
if (isQualificationFeatureName(name)) return QUALIFICATIONS_FILTER_NAME;
if (isTenureFeatureName(name)) return TENURE_FILTER_NAME;
const poiFilterName = getPoiFilterName(name); const poiFilterName = getPoiFilterName(name);
if (poiFilterName) return poiFilterName; if (poiFilterName) return poiFilterName;
return name; return name;
@ -555,12 +649,24 @@ export function summarizeParams(queryString: string): string {
for (let i = 0; i < crimeParams.length; i++) { for (let i = 0; i < crimeParams.length; i++) {
filterNames.push(SPECIFIC_CRIMES_FILTER_NAME); filterNames.push(SPECIFIC_CRIMES_FILTER_NAME);
} }
for (const entry of crimeSeverityParams) {
const colonIdx = entry.indexOf(':');
const featureName = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
const severityFilterName = getCrimeSeverityFilterName(featureName);
if (severityFilterName) filterNames.push(severityFilterName);
}
for (let i = 0; i < voteShareParams.length; i++) { for (let i = 0; i < voteShareParams.length; i++) {
filterNames.push(ELECTION_VOTE_SHARE_FILTER_NAME); filterNames.push(ELECTION_VOTE_SHARE_FILTER_NAME);
} }
for (let i = 0; i < ethnicityParams.length; i++) { for (let i = 0; i < ethnicityParams.length; i++) {
filterNames.push(ETHNICITIES_FILTER_NAME); filterNames.push(ETHNICITIES_FILTER_NAME);
} }
for (let i = 0; i < qualificationParams.length; i++) {
filterNames.push(QUALIFICATIONS_FILTER_NAME);
}
for (let i = 0; i < tenureParams.length; i++) {
filterNames.push(TENURE_FILTER_NAME);
}
for (let i = 0; i < amenityDistanceParams.length; i++) { for (let i = 0; i < amenityDistanceParams.length; i++) {
filterNames.push(POI_DISTANCE_FILTER_NAME); filterNames.push(POI_DISTANCE_FILTER_NAME);
} }

View file

@ -0,0 +1,80 @@
import type { FeatureMeta } from '../types';
/** i18n keys (typed for the strict `t()`) usable as a variant dropdown label. */
type VariantDropdownLabelKey =
| 'filters.crimeType'
| 'filters.qualificationLevel'
| 'filters.tenureType';
/** i18n keys (typed for the strict `t()`) usable as a window-toggle label. */
type VariantWindowLabelKey =
| 'filters.crimeWindow'
| 'filters.crimeWindow7y'
| 'filters.crimeWindow2y';
/** One option in a variant filter's secondary window/period toggle. */
export interface VariantWindowOption {
/** Stable id encoded inside the feature name (e.g. "7y"). */
id: string;
/** i18n key for the toggle button label. */
labelKey: VariantWindowLabelKey;
}
/**
* Optional secondary axis for a variant filter: the same variant measured over
* a different time window (e.g. crime rates over 7 vs 2 years). The dropdown
* still picks the variant; this toggle picks the window. Both ultimately select
* a single backend feature name, so switching either re-points the filter key.
*/
export interface VariantWindowConfig {
/** Toggle options in display order. */
options: VariantWindowOption[];
/** Optional i18n key for a small label above the toggle. */
labelKey?: VariantWindowLabelKey;
/** Window id of a feature name (e.g. "7y"), or null if unrecognized. */
getWindow: (featureName: string) => string | null;
/** The same variant's feature name in a different window. */
withWindow: (featureName: string, windowId: string) => string;
}
/**
* Shared shape for the "pick one backend feature variant, filter its range"
* family of client-side aggregate filters. Several Census/police breakdowns
* (specific crimes, qualifications, ) are dozens of individual percentage
* features that would clutter the filter browser as separate sliders. Instead
* each is folded into a single named filter whose card carries a dropdown to
* choose the variant, reusing one component ([`VariantFilterCard`]).
*
* Each filter library (e.g. `crime-filter`, `qualification-filter`) exports a
* config so the card stays variant-agnostic.
*/
export interface VariantFilterConfig {
/** Display name of the aggregate filter, e.g. "Specific crimes". */
filterName: string;
/**
* Backend feature names enumerating the dropdown options, in display order.
* With a [`window`] config these are the canonical (default-window) names;
* the card re-points each to the currently selected window.
*/
featureNames: readonly string[];
/** i18n key for the dropdown label, e.g. "filters.crimeType". */
dropdownLabelKey: VariantDropdownLabelKey;
/** Synthetic [`FeatureMeta`] used for the aggregate filter's own label/info. */
getFilterMeta: (features: FeatureMeta[]) => FeatureMeta;
/** First selectable variant present in `features`, or null if none. */
getDefaultFeatureName: (features: FeatureMeta[]) => string | null;
/** Backend feature name for a filter key (or a bare feature name). */
getFeatureName: (name: string) => string | null;
/** Rewrite a filter key to point at a different variant, keeping its id. */
replaceFilterKeySelection: (key: string, featureName: string) => string;
/** Clamp a [min, max] range into the selected feature's bounds. */
clampRange: (value: [number, number], feature?: FeatureMeta) => [number, number];
/**
* Server-translatable source value for a dropdown option's label; the card
* renders `ts(...)` of it. Defaults to the feature name itself. Useful with a
* [`window`] toggle so the label can be the bare variant (no window suffix).
*/
getOptionLabelSource?: (featureName: string) => string;
/** Optional secondary window/period toggle (e.g. 7- vs 2-year crime rates). */
window?: VariantWindowConfig;
}

View file

@ -316,26 +316,17 @@ export interface CrimeYearStats {
} }
export interface CrimeAreaAverage { export interface CrimeAreaAverage {
/** Full rate-feature name (e.g. "Burglary (per 1k/yr, 7y)"). */ /** Full crime-feature name (e.g. "Burglary (/yr, 7y)"). */
name: string; name: string;
/** Exact national mean rate. Preferred over the histogram-bin national /** Exact national mean count. Preferred over the histogram-bin national
* average for crime so all reference numbers share one estimator. */ * average for crime so all reference numbers share one estimator. */
national?: number; national?: number;
/** Mean rate across the selection's outcode. */ /** Mean count across the selection's outcode. */
outcode?: number; outcode?: number;
/** Mean rate across the selection's postcode sector. */ /** Mean count across the selection's postcode sector. */
sector?: number; sector?: number;
} }
export interface CrimeRawStats {
/** Bare crime type (e.g. "Burglary"). */
name: string;
/** Mean recorded incidents/yr over the last 7 years. */
per_yr_7y?: number;
/** Mean recorded incidents/yr over the last 2 years. */
per_yr_2y?: number;
}
/** One individual police.uk crime, from /api/crime-records. */ /** One individual police.uk crime, from /api/crime-records. */
export interface CrimeIncident { export interface CrimeIncident {
/** "YYYY-MM". */ /** "YYYY-MM". */
@ -352,6 +343,11 @@ export interface CrimeRecordsResponse {
records: CrimeIncident[]; records: CrimeIncident[];
total: number; total: number;
offset: number; offset: number;
/**
* Server flag meaning "more pages exist beyond this one". The client derives
* pagination from `total` vs `records.length`, so this is currently not
* surfaced in the UI (kept to mirror the server response shape).
*/
truncated: boolean; truncated: boolean;
} }
@ -386,12 +382,9 @@ export interface HexagonStatsResponse {
/** Postcode sector (e.g. "E14 2") of the selection's central postcode, when /** Postcode sector (e.g. "E14 2") of the selection's central postcode, when
* sector crime averages are available for it. */ * sector crime averages are available for it. */
crime_sector?: string; crime_sector?: string;
/** Per-rate-feature average rates across the central postcode's outcode and /** Per-crime-feature average counts across the central postcode's outcode and
* sector, shown alongside the national average for each crime metric. */ * sector, shown alongside the national average for each crime metric. */
crime_area_averages?: CrimeAreaAverage[]; crime_area_averages?: CrimeAreaAverage[];
/** Raw (un-normalised) recorded incidents/yr per crime type, shown beside the
* normalised rate. Display-only. */
crime_raw?: CrimeRawStats[];
/** Total individual crime records (last 7 years) across the selection's /** Total individual crime records (last 7 years) across the selection's
* postcodes the count behind the "individual crimes" list. */ * postcodes the count behind the "individual crimes" list. */
crime_total_records?: number; crime_total_records?: number;

View file

@ -0,0 +1,212 @@
"""Precompute per-outcode / per-postcode-sector / national mean headline crime
counts for the right pane's area comparison.
The right pane shows each crime metric next to its area context: the mean
average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
within a postcode (the merge keys it on the postcode), so each postcode
contributes its single value weighted by how many properties sit in it keeping
every scope on the same property-weighted basis as the per-selection mean, so the
four numbers (this selection / sector / outcode / nation) are directly
comparable. The national figure here is an EXACT property-weighted mean, which is
why it overrides the upward-biased histogram-bin national average for crime.
This used to be recomputed inside the server on every boot from the loaded
property matrix. It is a pure function of the two merge outputs, so it belongs in
the data build; the server now just loads the parquet this writes. Reading the
crime values from ``postcode.parquet`` and the per-postcode property weights from
``properties.parquet`` mirrors exactly the two inputs the server loads, so the
result matches what the server used to compute (minus its u16 quantization loss).
Output schema one row per area:
scope : ``"national"`` | ``"outcode"`` | ``"sector"``
area : the outcode (``"E14"``) / sector (``"E14 2"``);
``""`` for the single national row
``<type> (/yr, 7y|2y)`` : Float32 property-weighted mean crime count per year
(null = the scope has no data for that type)
"""
import argparse
from pathlib import Path
import polars as pl
# Filterable crime columns are the average-annual incident counts and carry this
# marker in postcode.parquet (e.g. "Burglary (/yr, 7y)"). We average those. The
# full column name is kept; the server discovers and keys area averages by the
# same names.
COUNT_MARKER = " (/yr, "
# `scope` discriminator values. The server's loader dispatches on these.
SCOPE_NATIONAL = "national"
SCOPE_OUTCODE = "outcode"
SCOPE_SECTOR = "sector"
# Area label on the national row — it spans the whole country, so it has no code.
NATIONAL_AREA = ""
# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
# "E14 2DG").
POSTCODE_COLUMN = "Postcode"
# Internal weight / split columns dropped before write.
_WEIGHT_COLUMN = "_weight"
_OUTCODE_COLUMN = "_outcode"
_SECTOR_COLUMN = "_sector"
def _crime_columns(columns: list[str]) -> list[str]:
crime_cols = [name for name in columns if COUNT_MARKER in name]
if not crime_cols:
raise ValueError(
f"postcode parquet has no '*{COUNT_MARKER}*' crime count columns to average"
)
return crime_cols
def _weighted_mean(column: str) -> pl.Expr:
"""Property-weighted mean of ``column`` that excludes nulls from BOTH the
value sum and the weight.
A null crime value is a genuine gap (the postcode's police force published no
usable data), not zero crime, so it must dilute neither the numerator nor the
denominator exactly as the server's former estimator skipped NaN values.
Yields null when no postcode in the group has data for this type.
"""
weight = pl.col(_WEIGHT_COLUMN)
numerator = (pl.col(column) * weight).sum()
denominator = weight.filter(pl.col(column).is_not_null()).sum()
return (
pl.when(denominator > 0)
.then(numerator / denominator)
.otherwise(None)
.cast(pl.Float32)
.alias(column)
)
def compute_area_crime_averages(
postcodes: pl.LazyFrame, properties: pl.LazyFrame
) -> pl.DataFrame:
"""Build the national / per-outcode / per-sector crime-average table.
``postcodes`` is the merge's postcode output (one row per active postcode,
carrying the ``"* (/yr, *)"`` crime count columns); ``properties`` is the merge's
per-property output, used only to weight each postcode by its property count.
"""
crime_cols = _crime_columns(postcodes.collect_schema().names())
# Property weight per postcode = how many property rows the server indexes
# under it. The inner join keeps only postcodes that actually carry
# properties, matching the server's per-postcode row index (a postcode with
# no properties never contributed to any average).
weights = properties.group_by(POSTCODE_COLUMN).agg(pl.len().alias(_WEIGHT_COLUMN))
# Outcode / sector of the spaced `pcds` postcode, matching the server's
# postcode_outcode / postcode_sector (split on the single space; sector =
# outcode + space + first inward character). Null where the form has no
# inward code, so such rows drop out of the per-area groups.
parts = pl.col(POSTCODE_COLUMN).str.splitn(" ", 2).struct
outward = parts.field("field_0")
inward = parts.field("field_1")
base = (
postcodes.select(POSTCODE_COLUMN, *crime_cols)
.join(weights, on=POSTCODE_COLUMN, how="inner")
.with_columns(
pl.when(inward.is_not_null())
.then(outward)
.otherwise(None)
.alias(_OUTCODE_COLUMN),
pl.when(inward.str.len_chars() >= 1)
.then(outward + pl.lit(" ") + inward.str.slice(0, 1))
.otherwise(None)
.alias(_SECTOR_COLUMN),
)
)
mean_exprs = [_weighted_mean(column) for column in crime_cols]
national = base.select(
pl.lit(SCOPE_NATIONAL).alias("scope"),
pl.lit(NATIONAL_AREA).alias("area"),
*mean_exprs,
)
by_outcode = (
base.drop_nulls(_OUTCODE_COLUMN)
.group_by(_OUTCODE_COLUMN)
.agg(mean_exprs)
.select(
pl.lit(SCOPE_OUTCODE).alias("scope"),
pl.col(_OUTCODE_COLUMN).alias("area"),
*crime_cols,
)
)
by_sector = (
base.drop_nulls(_SECTOR_COLUMN)
.group_by(_SECTOR_COLUMN)
.agg(mean_exprs)
.select(
pl.lit(SCOPE_SECTOR).alias("scope"),
pl.col(_SECTOR_COLUMN).alias("area"),
*crime_cols,
)
)
result = pl.concat([national, by_outcode, by_sector], how="vertical").collect(
engine="streaming"
)
# Drop per-area rows where every crime type is null: the server only created
# a map entry once a scope had at least one finite value, so an all-null
# outcode/sector reported no code at all. The national row is always kept (it
# always has data, and is emitted even for areas absent from both maps).
has_any = pl.any_horizontal(pl.col(column).is_not_null() for column in crime_cols)
return result.filter((pl.col("scope") == SCOPE_NATIONAL) | has_any)
def main() -> None:
parser = argparse.ArgumentParser(
description=(
"Precompute national / per-outcode / per-sector mean headline crime "
"counts from the merge outputs"
)
)
parser.add_argument(
"--postcodes",
type=Path,
required=True,
help="postcode.parquet (area features, incl. the '* (/yr, *)' crime columns)",
)
parser.add_argument(
"--properties",
type=Path,
required=True,
help="properties.parquet (per-property rows; supplies postcode property weights)",
)
parser.add_argument(
"--output",
type=Path,
required=True,
help="Output area_crime_averages.parquet path",
)
args = parser.parse_args()
result = compute_area_crime_averages(
pl.scan_parquet(args.postcodes), pl.scan_parquet(args.properties)
)
outcodes = result.filter(pl.col("scope") == SCOPE_OUTCODE).height
sectors = result.filter(pl.col("scope") == SCOPE_SECTOR).height
print(
f"Area crime averages: {result.height} rows "
f"({outcodes} outcodes, {sectors} sectors, "
f"{len(_crime_columns(result.columns))} crime types)"
)
args.output.parent.mkdir(parents=True, exist_ok=True)
result.write_parquet(args.output, compression="zstd")
print(f"Saved to {args.output}")
if __name__ == "__main__":
main()

View file

@ -2,22 +2,26 @@
Instead of attributing each incident to its published LSOA code, this transform Instead of attributing each incident to its published LSOA code, this transform
counts the anonymised incident *points* that fall within ``buffer_m`` (default counts the anonymised incident *points* that fall within ``buffer_m`` (default
100m) of each postcode's boundary polygon (the polygon buffered outward). A point 50m) of each postcode's boundary polygon (the polygon buffered outward). A point
inside several overlapping buffers counts for each postcode -- the same inside several overlapping buffers counts for each postcode -- the same
multiplicity the tree-density filter uses for features near more than one multiplicity the tree-density filter uses for features near more than one
postcode. The wide 100m buffer deliberately smooths police.uk's snap-to-grid postcode. The 50m buffer deliberately smooths police.uk's snap-to-grid
coordinates, which would otherwise make the count hypersensitive to which side of coordinates, which would otherwise make the count hypersensitive to which side of
a narrow line a shared "map point" anchor happened to land on. a narrow line a shared "map point" anchor happened to land on.
Counts are **area-normalised**: each postcode's count is divided by its buffered One figure is produced for every postcode and crime type, averaged over the
catchment area and rescaled by the median catchment area, so the metric reflects **last 7 years** and the **last 2 years**:
crime *density* rather than how much ground the buffer sweeps (a median-sized
catchment is left unchanged; a large rural postcode is no longer inflated simply * the **average annual incident count** -- ``sum(counts in covered window) * 12 /
for covering more of the map). Normalising by the buffered area -- the region covered_months`` -- the raw, absolute number of recorded incidents per year,
that actually collects points -- rather than the raw polygon keeps tiny unit with no per-area or per-capita normalisation. A covered postcode with no
postcodes from being over-inflated by the fixed buffer-ring floor. NOTE: this is incidents of a type gets ``0``; a postcode whose force never published in the
an incident *density of the surrounding streets*, not a per-resident risk -- window, or whose geometry is unusable, gets a *null* (unknown, never a zero).
zero-resident commercial centres (Soho, retail parks) legitimately rank high.
This figure is what the server exposes as the filterable crime feature -- the
headline metric in the right pane. (An earlier version divided it by an ambient
daytime population to get a per-1,000-people rate; that was hard to read, so the
absolute per-year count is now used directly.)
**Force-coverage calendar.** police.uk has multi-year publication gaps for whole **Force-coverage calendar.** police.uk has multi-year publication gaps for whole
forces (Greater Manchester has published nothing between 2019-07 and the present forces (Greater Manchester has published nothing between 2019-07 and the present
@ -29,28 +33,25 @@ computed against the months the postcode's own force actually published:
matched it (BTP, which reports nationwide, is excluded from the vote); matched it (BTP, which reports nationwide, is excluded from the vote);
postcodes with no incidents inherit their outcode's majority force, then the postcodes with no incidents inherit their outcode's majority force, then the
national modal force. national modal force.
* The headline ``"{type} (avg/yr)"`` is the POOLED annualised rate over the * A window's average pools the counts over the force's *covered* months inside
force's covered months: ``sum(counts in covered years) * 12 / covered_months``. that window and annualises by those months, so a coverage gap shrinks the data
Years in which the force published nothing contribute neither incidents nor rather than reading as a low-crime dip.
months, so a coverage gap no longer reads as a low-crime period. (Pooling over * The by-year series only emits bars for years with at least ``min_bar_months``
covered months also fixes the old "divide by years-with-incidents" headline, covered months (default 6).
which inflated sporadic categories by up to ~15x.)
* The by-year series only emits bars for years with at least
``min_bar_months`` covered months (default 6): annualising a single observed
month x12 produced misleading spikes. Each bar is scaled by the force's
covered months in that year, not the global month calendar.
* ``covered_years`` (list[struct{year, months}]) is written for every postcode * ``covered_years`` (list[struct{year, months}]) is written for every postcode
so the server can tell "covered, zero crime" (year listed, no bar) from "no so the server can tell "covered, zero crime" from "no data".
data" (year absent) instead of charting gaps as zeros.
* Postcodes whose boundary buffer is unusable (broken geometry) get null * Postcodes whose boundary buffer is unusable (broken geometry) get null
headline columns and an empty ``covered_years`` -- unknown, not zero. figures and an empty ``covered_years`` -- unknown, not zero.
Outputs mirror the old LSOA transform's shape but are keyed on ``postcode``: Outputs, all keyed on ``postcode``:
* ``crime_by_postcode.parquet`` -- ``postcode`` + ``"{type} (avg/yr)"`` columns. * ``crime_by_postcode.parquet`` -- ``"{type} (/yr, 7y)"`` / ``"{type} (/yr, 2y)"``
* ``crime_by_postcode_by_year.parquet`` -- one row per postcode: ``postcode`` + average-annual-count columns (the filterable crime features).
``covered_years`` + nested ``"{type} (by year)"`` ``list[struct{year, count}]`` * ``crime_by_postcode_by_year.parquet`` -- ``covered_years`` + nested
columns, with Serious/Minor rollups. ``"{type} (by year)"`` ``list[struct{year, count}]`` per-year raw counts.
* ``crime_records.parquet`` -- one row per counted incident over the last 7
years (``postcode`` + month/type/location/outcome/lat/lon), sorted by
postcode so the server can slice each postcode's incidents directly.
Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
points", not true locations, and a share of rows have no coordinate at all points", not true locations, and a share of rows have no coordinate at all
@ -63,6 +64,7 @@ from __future__ import annotations
import argparse import argparse
import re import re
import sys import sys
import tempfile
from pathlib import Path from pathlib import Path
import numpy as np import numpy as np
@ -80,22 +82,36 @@ from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
# Serious types first so column order is stable and self-documenting. # Serious types first so column order is stable and self-documenting.
ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
# Output type axis = the 14 leaf types plus the two rollups, in that order.
ROLLUP_TYPES: tuple[str, ...] = ("Serious crime", "Minor crime")
ALL_OUTPUT_TYPES: tuple[str, ...] = ALL_CRIME_TYPES + ROLLUP_TYPES
DEFAULT_BUFFER_M = 100.0 DEFAULT_BUFFER_M = 50.0
MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$") MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$") STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
# Trailing-window definitions, in (label, years) form. Each window's average is
# pooled over the force's covered months inside the window; one average-annual-
# count column (`(/yr, <label>)`) — the filterable crime feature — is emitted per
# window.
WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))
# Per-incident records cover the longest window's calendar years so the
# "individual crimes" list reconciles exactly with the headline counts: every
# record is an incident inside that window and vice versa. The headline counts
# are pooled per calendar year, so the records window must be calendar-year
# aligned too -- a rolling month span (e.g. a fixed 84 months) would include
# incidents from a year the headline excludes when the latest month is mid-year.
RECORDS_WINDOW_YEARS = max(win_years for _, win_years in WINDOWS)
# Minimum covered months for a year to get a by-year chart bar (and to be # Minimum covered months for a year to get a by-year chart bar (and to be
# listed in `covered_years`). Annualising fewer observed months (x12 from a # listed in `covered_years`). Annualising fewer observed months (x12 from a
# single month at the worst) produces bars dominated by noise, and the first # single month at the worst) produces bars dominated by noise. Six months keeps
# (2010: one month) and current partial year would otherwise always chart as # the annualisation factor <= 2.
# spikes/dips. Six months keeps the annualisation factor <= 2.
MIN_BAR_MONTHS = 6 MIN_BAR_MONTHS = 6
# Forces that report nationwide rather than policing a territory. They never # Forces that report nationwide rather than policing a territory. They never
# define a postcode's home force (their publication calendar says nothing about # define a postcode's home force, but their incidents still count.
# whether the *territorial* force covering the postcode published), but their
# incidents still count toward whichever postcodes they fall in.
NON_TERRITORIAL_FORCES = frozenset({"btp"}) NON_TERRITORIAL_FORCES = frozenset({"btp"})
COVERAGE_COLUMN = "covered_years" COVERAGE_COLUMN = "covered_years"
@ -106,8 +122,14 @@ LON_BOUNDS = (-9.5, 2.5)
LAT_BOUNDS = (49.0, 61.5) LAT_BOUNDS = (49.0, 61.5)
# Read CSVs in chunks of files to bound peak memory while keeping the STRtree # Read CSVs in chunks of files to bound peak memory while keeping the STRtree
# query vectorised over a useful number of points. # query vectorised over a useful number of points. Kept modest because the
_CSV_BATCH = 64 # in-window batches also materialise the per-incident record strings.
_CSV_BATCH = 32
def crime_column(type_name: str, window: str) -> str:
"""Filterable average-annual-count column, e.g. ``"Burglary (/yr, 7y)"``."""
return f"{type_name} (/yr, {window})"
def _force_calendar( def _force_calendar(
@ -115,12 +137,9 @@ def _force_calendar(
) -> tuple[list[int], list[str], np.ndarray]: ) -> tuple[list[int], list[str], np.ndarray]:
"""Derive the per-force publication calendar from the CSV paths. """Derive the per-force publication calendar from the CSV paths.
Each police.uk file lives under ``{crime_dir}/{YYYY-MM}/{YYYY-MM}-{force}- File presence IS the coverage signal: a (force, month) with no file
street.csv`` and holds that force's incidents for that month, so file published nothing. Returns the sorted distinct years, the force slugs
presence IS the coverage signal: a (force, month) with no file published (sorted), and ``months_in_year_force`` of shape (n_forces, n_years).
nothing. Returns the sorted distinct years, the force slugs (sorted), and
``months_in_year_force`` of shape (n_forces, n_years) -- how many months
each force published in each year.
""" """
month_force: set[tuple[str, str]] = set() month_force: set[tuple[str, str]] = set()
for path in csvs: for path in csvs:
@ -142,9 +161,6 @@ def _force_calendar(
for month, force in month_force: for month, force in month_force:
months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1 months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1
# Surface coverage gaps loudly: any territorial force missing months inside
# the global publication window is exactly the data hole the coverage
# masking exists for.
all_months = {month for month, _ in month_force} all_months = {month for month, _ in month_force}
for force in forces: for force in forces:
published = {m for m, f in month_force if f == force} published = {m for m, f in month_force if f == force}
@ -159,22 +175,25 @@ def _force_calendar(
def _build_tree( def _build_tree(
polygons: np.ndarray, buffer_m: float polygons: np.ndarray, buffer_m: float
) -> tuple[np.ndarray, shapely.STRtree]: ) -> tuple[shapely.STRtree, np.ndarray]:
"""Buffer postcode polygons outward by ``buffer_m`` and index them. """Buffer postcode polygons outward by ``buffer_m`` and index them.
Buffer index == postcode index. Geometries that fail to buffer are replaced Buffer index == postcode index. Returns the STRtree and a ``usable`` boolean
with an empty polygon so the index stays aligned; they simply never match. mask: geometries that fail to buffer are replaced with an empty polygon (so
the index stays aligned and they never match) and marked unusable, so their
crime picture is reported as unknown rather than zero.
""" """
buffers = shapely.buffer(polygons, buffer_m, quad_segs=8) buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers) broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
if broken.any(): if broken.any():
print(f" {int(broken.sum()):,} postcode buffers unusable; left empty") print(f" {int(broken.sum()):,} postcode buffers unusable; left empty")
buffers[broken] = shapely.from_wkt("POLYGON EMPTY") buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
return buffers, shapely.STRtree(buffers) return shapely.STRtree(buffers), ~broken
def _accumulate_counts( def _accumulate_counts(
csvs: list[Path], csvs: list[Path],
postcodes: np.ndarray,
tree: shapely.STRtree, tree: shapely.STRtree,
type_to_idx: dict[str, int], type_to_idx: dict[str, int],
year_to_idx: dict[int, int], year_to_idx: dict[int, int],
@ -182,35 +201,49 @@ def _accumulate_counts(
transformer: Transformer, transformer: Transformer,
counts: np.ndarray, counts: np.ndarray,
force_votes: np.ndarray, force_votes: np.ndarray,
records_shard_dir: Path | None,
records_min_ym: int,
) -> None: ) -> None:
"""Stream the crime CSVs, counting points-in-buffer per (postcode, type, year). """Stream the crime CSVs, counting points-in-buffer per (postcode, type, year).
Also accumulates ``force_votes`` (n_postcodes, n_forces): how many matched Also accumulates ``force_votes`` (n_postcodes, n_forces) for the home-force
incidents each force's files contributed to each postcode, which later election and, when ``records_shard_dir`` is set, writes one parquet shard per
elects the postcode's home force for the coverage calendar. batch holding every counted (incident, postcode) pair whose month is within
the records window (month index >= ``records_min_ym``).
""" """
# Type overrides only for the columns we ever read; LSOA is not stored.
schema = { schema = {
"Longitude": pl.Float64, "Longitude": pl.Float64,
"Latitude": pl.Float64, "Latitude": pl.Float64,
"Month": pl.Utf8, "Month": pl.Utf8,
"Crime type": pl.Utf8, "Crime type": pl.Utf8,
"Location": pl.Utf8,
"Last outcome category": pl.Utf8,
} }
years = list(year_to_idx) years = list(year_to_idx)
total_points = 0 total_points = 0
total_matches = 0 total_matches = 0
total_dropped = 0 total_dropped = 0
total_records = 0
unknown_type_counts: dict[str, int] = {} unknown_type_counts: dict[str, int] = {}
for start in range(0, len(csvs), _CSV_BATCH): for start in range(0, len(csvs), _CSV_BATCH):
batch = csvs[start : start + _CSV_BATCH] batch = csvs[start : start + _CSV_BATCH]
# The source file identifies the publishing force (police.uk has no
# force column with consistent naming); map each path back to its
# force index for the home-force vote.
path_to_fidx = {} path_to_fidx = {}
batch_max_ym = -1
for path in batch: for path in batch:
m = STREET_CSV_NAME_RE.fullmatch(path.name) m = STREET_CSV_NAME_RE.fullmatch(path.name)
if m is not None and m.group(2) in force_to_idx: if m is not None:
path_to_fidx[str(path)] = force_to_idx[m.group(2)] ym = m.group(1)
batch_max_ym = max(batch_max_ym, int(ym[:4]) * 12 + int(ym[5:7]) - 1)
if m.group(2) in force_to_idx:
path_to_fidx[str(path)] = force_to_idx[m.group(2)]
# The per-incident record strings (Location, outcome) are by far the
# heaviest columns; read them only for batches that fall inside the
# records window, so the ~50% of pre-window months cost nothing extra.
want_records = records_shard_dir is not None and batch_max_ym >= records_min_ym
record_cols = ["Location", "Last outcome category"] if want_records else []
frame = ( frame = (
pl.scan_csv( pl.scan_csv(
batch, batch,
@ -218,12 +251,12 @@ def _accumulate_counts(
ignore_errors=True, ignore_errors=True,
include_file_paths="_source_path", include_file_paths="_source_path",
) )
.select("Longitude", "Latitude", "Month", "Crime type", "_source_path") .select(
# strict=False: a single malformed Month drops only that row instead "Longitude", "Latitude", "Month", "Crime type", *record_cols, "_source_path"
# of aborting the whole build (a non-numeric year becomes null and is )
# filtered out by the year membership check below).
.with_columns( .with_columns(
pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year") pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year"),
pl.col("Month").str.slice(5, 2).cast(pl.Int32, strict=False).alias("_mm"),
) )
.filter( .filter(
pl.col("Longitude").is_not_null() pl.col("Longitude").is_not_null()
@ -234,14 +267,11 @@ def _accumulate_counts(
& (pl.col("Crime type") != "") & (pl.col("Crime type") != "")
& pl.col("year").is_in(years) & pl.col("year").is_in(years)
) )
# Canonicalise legacy pre-2014 crime-type names ("Violent crime", # year*12 + (month-1): an integer month index for window filtering.
# "Public disorder and weapons") to their current equivalents before .with_columns(
# indexing, so ~1.9M historical incidents are counted instead of (pl.col("year") * 12 + (pl.col("_mm").fill_null(1) - 1)).alias("month_index")
# dropped. `.replace` leaves current types unchanged. )
.with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES)) .with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
# Map crime types to indices with default=None so an unrecognised
# type yields a null index we can *report* rather than silently drop
# (the legacy LSOA path surfaced unknown types via its dynamic pivot).
.with_columns( .with_columns(
pl.col("Crime type") pl.col("Crime type")
.replace_strict(type_to_idx, default=None, return_dtype=pl.Int32) .replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
@ -253,7 +283,16 @@ def _accumulate_counts(
.replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32) .replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32)
.alias("fidx"), .alias("fidx"),
) )
.select("Longitude", "Latitude", "Crime type", "tidx", "yidx", "fidx") .select(
"Longitude",
"Latitude",
"Crime type",
*record_cols,
"month_index",
"tidx",
"yidx",
"fidx",
)
.collect(engine="streaming") .collect(engine="streaming")
) )
@ -273,19 +312,15 @@ def _accumulate_counts(
tidx = frame["tidx"].to_numpy() tidx = frame["tidx"].to_numpy()
yidx = frame["yidx"].to_numpy() yidx = frame["yidx"].to_numpy()
fidx = frame["fidx"].to_numpy() fidx = frame["fidx"].to_numpy()
month_index = frame["month_index"].to_numpy()
x, y = transformer.transform(lon, lat) x, y = transformer.transform(lon, lat)
finite = np.isfinite(x) & np.isfinite(y) finite = np.isfinite(x) & np.isfinite(y)
total_dropped += int((~finite).sum()) total_dropped += int((~finite).sum())
if not finite.any(): if not finite.any():
continue continue
x, y, tidx, yidx, fidx = ( x, y = x[finite], y[finite]
x[finite], tidx, yidx, fidx = tidx[finite], yidx[finite], fidx[finite]
y[finite],
tidx[finite],
yidx[finite],
fidx[finite],
)
total_points += x.size total_points += x.size
points = shapely.points(x, y) points = shapely.points(x, y)
@ -306,9 +341,26 @@ def _accumulate_counts(
) )
total_matches += point_index.size total_matches += point_index.size
if want_records:
total_records += _write_record_shard(
records_shard_dir,
start,
postcodes,
point_index,
postcode_index,
month_index[finite],
frame["Crime type"].to_numpy()[finite],
frame["Location"].to_numpy()[finite],
frame["Last outcome category"].to_numpy()[finite],
lon[finite],
lat[finite],
records_min_ym,
)
print( print(
f" files {start + len(batch):,}/{len(csvs):,}: " f" files {start + len(batch):,}/{len(csvs):,}: "
f"{total_points:,} located points, {total_matches:,} postcode matches" f"{total_points:,} located points, {total_matches:,} postcode matches"
+ (f", {total_records:,} records" if records_shard_dir is not None else "")
) )
if total_dropped: if total_dropped:
@ -329,19 +381,65 @@ def _accumulate_counts(
) )
def _write_record_shard(
records_shard_dir: Path,
start: int,
postcodes: np.ndarray,
point_index: np.ndarray,
postcode_index: np.ndarray,
month_index: np.ndarray,
crime_type: np.ndarray,
location: np.ndarray,
outcome: np.ndarray,
lon: np.ndarray,
lat: np.ndarray,
records_min_ym: int,
) -> int:
"""Write one parquet shard of (incident, postcode) records for this batch.
Each matched pair becomes a row -- the same multiplicity as the count -- so a
postcode's records are exactly the incidents that made up its counts. Only
incidents within the records window (month index >= ``records_min_ym``) are
kept. Returns the number of rows written.
"""
mi = month_index[point_index]
keep = mi >= records_min_ym
if not keep.any():
return 0
pidx = point_index[keep]
# Build the string columns from Python lists (.tolist()) rather than numpy
# object arrays: an all-null Location/outcome slice is an all-None object
# array that polars cannot cast to String, whereas a list of str|None infers
# a nullable String column cleanly.
shard = pl.DataFrame(
{
"postcode": postcodes[postcode_index[keep]].astype(str),
"month_index": mi[keep].astype(np.int32),
"crime_type": crime_type[pidx].astype(str),
"location": location[pidx].tolist(),
"outcome": outcome[pidx].tolist(),
"lat": lat[pidx].astype(np.float32),
"lon": lon[pidx].astype(np.float32),
},
schema_overrides={
"postcode": pl.String,
"crime_type": pl.String,
"location": pl.String,
"outcome": pl.String,
},
)
shard.write_parquet(
records_shard_dir / f"{start:08d}.parquet", compression="zstd"
)
return shard.height
def _assign_home_force( def _assign_home_force(
postcodes: np.ndarray, postcodes: np.ndarray,
force_votes: np.ndarray, force_votes: np.ndarray,
forces: list[str], forces: list[str],
) -> np.ndarray: ) -> np.ndarray:
"""Elect each postcode's home (territorial) force. """Elect each postcode's home (territorial) force by majority incident vote."""
Majority vote of matched incidents per publishing force; non-territorial
forces (BTP) are excluded from the vote because their calendar says nothing
about local coverage. Postcodes with no votes (no incidents ever, or
BTP-only) inherit the majority force of their outcode, then the national
modal force, so every postcode gets a coverage calendar.
"""
votes = force_votes.astype(np.int64, copy=True) votes = force_votes.astype(np.int64, copy=True)
for idx, force in enumerate(forces): for idx, force in enumerate(forces):
if force in NON_TERRITORIAL_FORCES: if force in NON_TERRITORIAL_FORCES:
@ -354,18 +452,15 @@ def _assign_home_force(
if not has_vote.any(): if not has_vote.any():
raise ValueError("No incidents matched any postcode; cannot assign forces") raise ValueError("No incidents matched any postcode; cannot assign forces")
# Outcode-majority fallback for postcodes with no (territorial) incidents.
outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object) outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object)
national_modal = int( national_modal = int(np.bincount(home[has_vote], minlength=len(forces)).argmax())
np.bincount(home[has_vote], minlength=len(forces)).argmax()
)
if (~has_vote).any(): if (~has_vote).any():
outcode_modal: dict[str, int] = {} outcode_modal: dict[str, int] = {}
voted_outcodes = outcodes[has_vote] voted_outcodes = outcodes[has_vote]
voted_home = home[has_vote] voted_home = home[has_vote]
for oc in np.unique(voted_outcodes): for oc in np.unique(voted_outcodes):
counts = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces)) tally = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
outcode_modal[oc] = int(counts.argmax()) outcode_modal[oc] = int(tally.argmax())
fallback = np.array( fallback = np.array(
[outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]], [outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]],
dtype=np.int32, dtype=np.int32,
@ -379,10 +474,82 @@ def _assign_home_force(
return home return home
def _window_annualised(
counts: np.ndarray,
months_in_year_force: np.ndarray,
home_fidx: np.ndarray,
usable: np.ndarray,
year_mask: np.ndarray,
) -> np.ndarray:
"""Raw annualised incidents/yr per (postcode, type) over a window of years.
For each force, the count is pooled over the force's covered months that fall
inside ``year_mask`` and annualised by those covered months. A covered
postcode with no incidents of a type gets 0; a postcode whose force never
published in the window, or whose geometry is unusable, gets NaN (unknown).
"""
n_pc, n_types = counts.shape[0], counts.shape[1]
avg = np.full((n_pc, n_types), np.nan, dtype=np.float64)
for f in range(months_in_year_force.shape[0]):
sel = home_fidx == f
if not sel.any():
continue
cov_months = months_in_year_force[f].astype(np.float64) * year_mask
denom = cov_months.sum()
if denom <= 0:
continue # force published nothing in this window; stays NaN
window_years = cov_months > 0
pooled = counts[sel][:, :, window_years].sum(axis=2, dtype=np.float64)
avg[sel] = pooled * 12.0 / denom
avg[~usable] = np.nan
return avg
def _append_rollups(avg14: np.ndarray) -> np.ndarray:
"""Append Serious/Minor rollup columns (sum of components) -> (n_pc, 16)."""
serious_idx = [ALL_CRIME_TYPES.index(t) for t in SERIOUS_CRIME_TYPES]
minor_idx = [ALL_CRIME_TYPES.index(t) for t in MINOR_CRIME_TYPES]
serious = avg14[:, serious_idx].sum(axis=1)
minor = avg14[:, minor_idx].sum(axis=1)
return np.column_stack([avg14, serious, minor])
def _write_crime_table(
postcodes: np.ndarray,
raw_by_window: dict[str, np.ndarray],
output_path: Path,
) -> None:
"""Write the average-annual-count parquet (the filterable crime features).
``raw_by_window[label]`` is the (n_postcodes, 16) average annual incident
count for that window (14 leaf types + 2 rollups). Each value is the raw,
absolute incidents/yr; a postcode with no usable data for a window keeps NaN
(written as null).
"""
data: dict[str, np.ndarray] = {"postcode": postcodes}
for label, _years in WINDOWS:
counts = np.round(raw_by_window[label], 1).astype(np.float32)
for type_idx, name in enumerate(ALL_OUTPUT_TYPES):
data[crime_column(name, label)] = counts[:, type_idx]
_write_nan_aware(data, output_path, "postcode crime average annual counts")
def _write_nan_aware(
data: dict[str, np.ndarray], output_path: Path, label: str
) -> None:
frame = pl.DataFrame(data)
value_cols = [c for c in frame.columns if c != "postcode"]
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
output_path.parent.mkdir(parents=True, exist_ok=True)
frame.write_parquet(output_path, compression="zstd")
print(f"Wrote {label}: {output_path} {frame.shape}")
def _rollup_long( def _rollup_long(
long: pl.DataFrame, types: tuple[str, ...], rollup_name: str long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
) -> pl.DataFrame: ) -> pl.DataFrame:
"""Sum per-year annualised counts across ``types`` into a single rollup.""" """Sum per-year counts across ``types`` into a single rollup."""
return ( return (
long.filter(pl.col("Crime type").is_in(list(types))) long.filter(pl.col("Crime type").is_in(list(types)))
.group_by("postcode", "year") .group_by("postcode", "year")
@ -392,108 +559,29 @@ def _rollup_long(
) )
def _write_avg_yr(
postcodes: np.ndarray,
counts: np.ndarray,
months_in_year_force: np.ndarray,
home_fidx: np.ndarray,
norm: np.ndarray,
output_path: Path,
) -> None:
"""Write ``postcode`` + ``"{type} (avg/yr)"`` density-normalised averages.
The headline is the POOLED annualised rate over the home force's covered
months: ``sum(counts in covered years) * 12 / covered_months``. Years the
force published nothing contribute neither incidents nor months, so a
coverage gap (e.g. Greater Manchester 2019-07 onwards) is excluded instead
of read as zero crime. Pooling over the full covered window -- rather than
averaging only over years a type happened to occur -- is what keeps a
single robbery-year from printing as a perennial robbery rate. Each
postcode's value is then multiplied by ``norm`` (median_area / buffered
catchment area) so the metric is a density rather than a footprint-inflated
raw count; postcodes with unusable geometry (norm == 0) are null, not 0.
"""
n_postcodes, n_types = counts.shape[0], counts.shape[1]
avg = np.full((n_postcodes, n_types), np.nan, dtype=np.float64)
for f in range(months_in_year_force.shape[0]):
sel = home_fidx == f
if not sel.any():
continue
cov_months = months_in_year_force[f].astype(np.float64)
denom = cov_months.sum()
if denom <= 0:
continue # force never published; stays null
covered_years = cov_months > 0
pooled = counts[sel][:, :, covered_years].sum(axis=2, dtype=np.float64)
avg[sel] = pooled * 12.0 / denom
avg *= norm[:, None]
avg[norm <= 0] = np.nan # unusable geometry: unknown, not zero
avg = np.round(avg, 1).astype(np.float32)
data: dict[str, np.ndarray] = {"postcode": postcodes}
for type_idx, name in enumerate(ALL_CRIME_TYPES):
data[f"{name} (avg/yr)"] = avg[:, type_idx]
# Serious/Minor rollup headlines = the exact SUM of their component (avg/yr)
# columns, so each rollup always equals the sum of the parts shown beside it
# and can never fall below one of its own components. All components share
# the postcode's pooled covered-month denominator, so the sum is itself the
# pooled rollup rate. Null components (unusable geometry) propagate to a
# null rollup.
for rollup_name, rollup_types in (
("Serious crime", SERIOUS_CRIME_TYPES),
("Minor crime", MINOR_CRIME_TYPES),
):
rollup_idx = [ALL_CRIME_TYPES.index(name) for name in rollup_types]
data[f"{rollup_name} (avg/yr)"] = np.round(
avg[:, rollup_idx].sum(axis=1), 1
).astype(np.float32)
frame = pl.DataFrame(data)
value_cols = [c for c in frame.columns if c != "postcode"]
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
output_path.parent.mkdir(parents=True, exist_ok=True)
frame.write_parquet(output_path, compression="zstd")
print(f"Wrote postcode crime averages: {output_path}")
def _write_by_year( def _write_by_year(
postcodes: np.ndarray, postcodes: np.ndarray,
counts: np.ndarray, counts: np.ndarray,
years: list[int], years: list[int],
months_in_year_force: np.ndarray, months_in_year_force: np.ndarray,
home_fidx: np.ndarray, home_fidx: np.ndarray,
norm: np.ndarray, usable: np.ndarray,
min_bar_months: int, min_bar_months: int,
output_path: Path, output_path: Path,
) -> None: ) -> None:
"""Write nested ``"{type} (by year)"`` series plus rollups and coverage. """Write nested ``"{type} (by year)"`` raw-count series plus rollups + coverage.
A bar is only emitted for (postcode, year)s where the postcode's home force A bar is only emitted for (postcode, year)s where the postcode's home force
published at least ``min_bar_months`` months -- annualising a thinner year published at least ``min_bar_months`` months. Bars are the raw annualised
(x12 from a single month at the extreme) charts noise, and a force-gap year count for that year (``count * 12 / covered_months``); unlike the headline
must chart as *no data*, not zero. Bars are scaled by the force's covered windows there is no per-capita normalisation here -- the chart shows incident
months in that year and area-normalised by the same ``norm`` factor as the volume over time. Every postcode gets a ``covered_years`` row so consumers can
headline so chart and headline stay mutually consistent. distinguish covered-but-crime-free years from coverage gaps.
Every postcode gets a row (the output is dense) carrying ``covered_years``
-- the list of {year, months} the home force published at least
``min_bar_months`` months -- so consumers can distinguish covered-but-
crime-free years (year listed, no bar => genuine zero) from coverage gaps
(year absent => unknown). Postcodes with unusable geometry get an empty
coverage list: their crime picture is unknown.
""" """
# (n_postcodes, n_years): covered months of each postcode's home force.
cov_pc_year = months_in_year_force[home_fidx, :] cov_pc_year = months_in_year_force[home_fidx, :]
usable = norm > 0
annual = np.round( annual = np.round(
counts.astype(np.float64) counts.astype(np.float64) * 12.0 / np.maximum(cov_pc_year[:, None, :], 1),
* 12.0
/ np.maximum(cov_pc_year[:, None, :], 1)
* norm[:, None, None],
1, 1,
) )
bar_ok = ( bar_ok = (
@ -506,9 +594,6 @@ def _write_by_year(
type_names = np.array(ALL_CRIME_TYPES, dtype=object) type_names = np.array(ALL_CRIME_TYPES, dtype=object)
year_values = np.array(years, dtype=np.int32) year_values = np.array(years, dtype=np.int32)
# Explicit schema: with full masking (e.g. every year below min_bar_months)
# the fancy-indexed numpy object arrays are empty and polars would infer
# Object columns, which breaks the rollup `is_in` below.
long = pl.DataFrame( long = pl.DataFrame(
{ {
"postcode": postcodes[pc_i].astype(str), "postcode": postcodes[pc_i].astype(str),
@ -532,8 +617,6 @@ def _write_by_year(
type_cols = [c for c in wide.columns if c != "postcode"] type_cols = [c for c in wide.columns if c != "postcode"]
wide = wide.rename({col: f"{col} (by year)" for col in type_cols}) wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
# Dense base: every postcode, with its home force's coverage calendar.
# Built per force (there are ~45) and joined on the force index.
coverage_per_force: list[list[dict[str, int]]] = [] coverage_per_force: list[list[dict[str, int]]] = []
for f in range(months_in_year_force.shape[0]): for f in range(months_in_year_force.shape[0]):
coverage_per_force.append( coverage_per_force.append(
@ -562,7 +645,6 @@ def _write_by_year(
dense = ( dense = (
base.join(coverage_frame, on="_fidx", how="left") base.join(coverage_frame, on="_fidx", how="left")
.with_columns( .with_columns(
# Unusable geometry: empty coverage -- the crime picture is unknown.
pl.when(pl.col("_usable")) pl.when(pl.col("_usable"))
.then(pl.col(COVERAGE_COLUMN)) .then(pl.col(COVERAGE_COLUMN))
.otherwise(pl.col(COVERAGE_COLUMN).list.head(0)) .otherwise(pl.col(COVERAGE_COLUMN).list.head(0))
@ -577,11 +659,47 @@ def _write_by_year(
print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}") print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}")
def _finalize_records(
records_shard_dir: Path, output_path: Path
) -> None:
"""Concatenate the per-batch record shards into one postcode-sorted parquet.
Sorting by postcode lets the server build a contiguous per-postcode slice
index. The sort runs on the streaming engine so it spills rather than holding
all ~40M rows in memory.
"""
shards = sorted(records_shard_dir.glob("*.parquet"))
output_path.parent.mkdir(parents=True, exist_ok=True)
if not shards:
pl.DataFrame(
schema={
"postcode": pl.String,
"month_index": pl.Int32,
"crime_type": pl.String,
"location": pl.String,
"outcome": pl.String,
"lat": pl.Float32,
"lon": pl.Float32,
}
).write_parquet(output_path, compression="zstd")
print(f"Wrote crime records: {output_path} (empty)")
return
(
pl.scan_parquet(shards)
.sort("postcode")
.sink_parquet(output_path, compression="zstd")
)
n = pl.scan_parquet(output_path).select(pl.len()).collect().item()
print(f"Wrote crime records: {output_path} ({n:,} rows)")
def transform_crime_spatial( def transform_crime_spatial(
crime_dir: Path, crime_dir: Path,
boundaries_dir: Path, boundaries_dir: Path,
output_path: Path, output_path: Path,
by_year_output_path: Path, by_year_output_path: Path,
records_output_path: Path,
buffer_m: float = DEFAULT_BUFFER_M, buffer_m: float = DEFAULT_BUFFER_M,
max_postcodes: int | None = None, max_postcodes: int | None = None,
max_files: int | None = None, max_files: int | None = None,
@ -594,6 +712,10 @@ def transform_crime_spatial(
csvs = csvs[:max_files] csvs = csvs[:max_files]
years, forces, months_in_year_force = _force_calendar(csvs) years, forces, months_in_year_force = _force_calendar(csvs)
latest_year = years[-1]
# Records cover the longest window's calendar years (January of its earliest
# year onward), so they reconcile exactly with the calendar-year headline.
records_min_ym = (latest_year - (RECORDS_WINDOW_YEARS - 1)) * 12
print( print(
f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces " f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces "
f"({years[0]}-{years[-1]})" f"({years[0]}-{years[-1]})"
@ -601,27 +723,10 @@ def transform_crime_spatial(
) )
postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes) postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
postcodes = np.asarray(postcodes)
print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...") print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
buffers, tree = _build_tree(polygons, buffer_m) tree, usable = _build_tree(polygons, buffer_m)
# Area-normalisation factor (median_area / catchment_area): divides out the
# size of each postcode's catchment so the count measures crime density, not
# how much ground the buffer sweeps. We normalise by the *buffered* area --
# the region that actually collects points -- rather than the raw polygon, so
# a tiny unit postcode isn't over-inflated by the fixed buffer-ring floor.
# Buffers are in EPSG:27700, so shapely.area is in m^2.
areas = shapely.area(buffers).astype(np.float64)
usable_area = np.isfinite(areas) & (areas > 0)
if not usable_area.any():
raise ValueError("No postcode buffers have a positive area to normalise by")
median_area = float(np.median(areas[usable_area]))
norm = np.zeros(len(postcodes), dtype=np.float64)
norm[usable_area] = median_area / areas[usable_area]
print(
f"Area-normalising to median catchment area {median_area:,.0f} m^2 "
f"({int(usable_area.sum()):,}/{len(areas):,} postcodes have usable area)"
)
type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)} type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
year_to_idx = {year: idx for idx, year in enumerate(years)} year_to_idx = {year: idx for idx, year in enumerate(years)}
@ -630,25 +735,50 @@ def transform_crime_spatial(
force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32) force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32)
transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True) transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
_accumulate_counts(
csvs, tree, type_to_idx, year_to_idx, force_to_idx, transformer, counts, force_votes
)
home_fidx = _assign_home_force(np.asarray(postcodes), force_votes, forces) with tempfile.TemporaryDirectory(
prefix="crime_records_", dir=records_output_path.parent
) as shard_dir_str:
shard_dir = Path(shard_dir_str)
_accumulate_counts(
csvs,
postcodes,
tree,
type_to_idx,
year_to_idx,
force_to_idx,
transformer,
counts,
force_votes,
shard_dir,
records_min_ym,
)
_write_avg_yr( home_fidx = _assign_home_force(postcodes, force_votes, forces)
postcodes, counts, months_in_year_force, home_fidx, norm, output_path
) # Per-window raw annualised averages (14 leaf types + Serious/Minor).
_write_by_year( raw_by_window: dict[str, np.ndarray] = {}
postcodes, for label, win_years in WINDOWS:
counts, year_mask = np.array(
years, [1.0 if y > latest_year - win_years else 0.0 for y in years]
months_in_year_force, )
home_fidx, avg14 = _window_annualised(
norm, counts, months_in_year_force, home_fidx, usable, year_mask
min_bar_months, )
by_year_output_path, raw_by_window[label] = _append_rollups(avg14)
)
_write_crime_table(postcodes, raw_by_window, output_path)
_write_by_year(
postcodes,
counts,
years,
months_in_year_force,
home_fidx,
usable,
min_bar_months,
by_year_output_path,
)
_finalize_records(shard_dir, records_output_path)
def main() -> None: def main() -> None:
@ -671,7 +801,7 @@ def main() -> None:
"--output", "--output",
type=Path, type=Path,
required=True, required=True,
help="Output parquet: postcode + '{type} (avg/yr)' columns", help="Output parquet: postcode + '{type} (/yr, <window>)' average-annual-count columns",
) )
parser.add_argument( parser.add_argument(
"--output-by-year", "--output-by-year",
@ -680,10 +810,10 @@ def main() -> None:
help="Output parquet: postcode + nested '{type} (by year)' columns", help="Output parquet: postcode + nested '{type} (by year)' columns",
) )
parser.add_argument( parser.add_argument(
"--buffer-m", "--output-records",
type=float, type=Path,
default=DEFAULT_BUFFER_M, required=True,
help="Outward buffer (metres) added to each postcode boundary", help="Output parquet: one row per counted incident (last 7 years), postcode-sorted",
) )
parser.add_argument( parser.add_argument(
"--max-postcodes", "--max-postcodes",
@ -705,15 +835,12 @@ def main() -> None:
) )
args = parser.parse_args() args = parser.parse_args()
if args.buffer_m <= 0:
raise SystemExit("--buffer-m must be greater than zero")
transform_crime_spatial( transform_crime_spatial(
crime_dir=args.input, crime_dir=args.input,
boundaries_dir=args.boundaries, boundaries_dir=args.boundaries,
output_path=args.output, output_path=args.output,
by_year_output_path=args.output_by_year, by_year_output_path=args.output_by_year,
buffer_m=args.buffer_m, records_output_path=args.output_records,
max_postcodes=args.max_postcodes, max_postcodes=args.max_postcodes,
max_files=args.max_files, max_files=args.max_files,
min_bar_months=args.min_bar_months, min_bar_months=args.min_bar_months,

View file

@ -0,0 +1,149 @@
"""Join the slim price estimates back onto properties.parquet.
Price estimation runs on ``price_inputs.parquet`` (built by ``property_base``
straight from epc_pp + arcgis, independently of merge's area features) and emits
``price_estimates.parquet`` the natural key (Postcode + coalesced address) plus
``Estimated current price`` / ``Est. price per sqm``. This step joins those two
columns onto properties.parquet to produce the file the server consumes.
Why the natural key
-------------------
Estimates and properties are built by separate runs, so a positional row index
would not line up. Instead both derive the key ``(Postcode, coalesce(register
address, EPC address))`` which is unique and non-null on the deduped dwelling
universe (see ``property_base._dedupe_collapsed_properties``) and identical on
both sides because both start from that same universe. So estimates map onto
properties 1:1 regardless of row order.
Re-running is safe: any pre-existing estimate columns are dropped first, and the
join is keyed (not positional), so a second run reproduces the same result. The
join refuses if any property has no estimate (the dwelling universes diverged,
e.g. a stale price_inputs vs a newer epc_pp) rather than silently leaving prices
null. Output is written to a temp file and atomically renamed.
"""
import argparse
from pathlib import Path
import polars as pl
from pipeline.transform.price_estimation.utils import (
ESTIMATE_COLUMNS,
JOIN_ADDRESS,
JOIN_KEYS,
join_address_expr,
)
def join_estimates(properties: Path, estimates_path: Path) -> int:
"""Augment ``properties`` in place with the estimate columns; return rows.
Joins the slim estimates onto properties by the natural key and atomically
replaces properties.parquet. Idempotent: any estimate columns already on the
file are dropped first.
"""
estimates = pl.scan_parquet(estimates_path)
est_cols = estimates.collect_schema().names()
missing = [c for c in (*JOIN_KEYS, *ESTIMATE_COLUMNS) if c not in est_cols]
if missing:
raise ValueError(f"{estimates_path}: missing columns {missing}")
stats = estimates.select(
n=pl.len(), unique=pl.struct(JOIN_KEYS).n_unique()
).collect(engine="streaming")
n_estimates, n_unique = stats["n"][0], stats["unique"][0]
if n_unique != n_estimates:
raise ValueError(
f"{estimates_path}: natural key {JOIN_KEYS} is not unique "
f"({n_estimates - n_unique:,} duplicate rows)"
)
n_properties = pl.scan_parquet(properties).select(pl.len()).collect().item()
# Drop any estimate columns already present (idempotent re-run) and attach the
# coalesced-address half of the natural key.
properties_keyed = (
pl.scan_parquet(properties)
.drop(ESTIMATE_COLUMNS, strict=False)
.with_columns(join_address_expr())
)
# Every property must have an estimate: estimates and properties come from the
# same dwelling universe, so a gap means a stale/foreign price_inputs (e.g.
# built from a different epc_pp). Fail loudly instead of nulling prices.
#
# This assumes properties.parquet contains ONLY epc_pp-derived dwellings, which
# is true for the production merge output. Running merge with --actual-listings
# appends listing seed rows whose (Postcode, address) keys are absent from
# price_inputs (built straight from epc_pp), which would trip the guard below.
# Enabling listing integration on the primary output therefore requires
# price_inputs to include those seed rows too.
unmatched = (
properties_keyed.select(JOIN_KEYS)
.join(estimates.select(JOIN_KEYS), on=JOIN_KEYS, how="anti")
.select(pl.len())
.collect(engine="streaming")
.item()
)
if unmatched:
raise ValueError(
f"{properties}: {unmatched:,} of {n_properties:,} properties have no "
"matching estimate; the price_inputs and properties dwelling universes "
"differ (regenerate price_inputs.parquet from the current epc_pp)."
)
# maintain_order="left" keeps properties in merge's row order; the unique key
# cannot fan the join out, so the row count is preserved.
result = properties_keyed.join(
estimates, on=JOIN_KEYS, how="left", maintain_order="left"
).drop(JOIN_ADDRESS)
tmp = properties.with_name(properties.name + ".tmp")
result.sink_parquet(tmp)
written = pl.scan_parquet(tmp).select(pl.len()).collect().item()
if written != n_properties:
tmp.unlink(missing_ok=True)
raise ValueError(
f"{properties}: join changed the row count "
f"({n_properties:,} -> {written:,})"
)
tmp.replace(properties)
return written
def main():
parser = argparse.ArgumentParser(
description="Join price_estimates.parquet onto properties.parquet"
)
parser.add_argument(
"--properties",
type=Path,
required=True,
help="properties.parquet (read, then overwritten with the estimate "
"columns joined in)",
)
parser.add_argument(
"--estimates",
type=Path,
required=True,
help="Slim price_estimates.parquet from price_estimation.estimate",
)
args = parser.parse_args()
written = join_estimates(args.properties, args.estimates)
size_mb = args.properties.stat().st_size / (1024 * 1024)
n_priced = (
pl.scan_parquet(args.properties)
.filter(pl.col("Estimated current price").is_not_null())
.select(pl.len())
.collect()
.item()
)
print(f"Wrote {args.properties} ({size_mb:.1f} MB)")
print(f" {written:,} rows, {n_priced:,} with an estimated current price")
if __name__ == "__main__":
main()

View file

@ -29,10 +29,16 @@ from pipeline.utils.fuzzy_join import (
normalize_address_key, normalize_address_key,
normalize_postcode_key, normalize_postcode_key,
) )
from pipeline.transform.property_base import (
MIN_FLOOR_AREA_M2,
_active_english_postcode_area,
_filter_to_active_english_postcodes,
build_property_base,
property_type_expr,
)
from pipeline.utils.normalize import drop_digit_tokens from pipeline.utils.normalize import drop_digit_tokens
from pipeline.utils.postcode_mapping import build_postcode_mapping from pipeline.utils.postcode_mapping import build_postcode_mapping
MIN_FLOOR_AREA_M2 = 10
CONSERVATION_AREA_FEATURE = "Within conservation area" CONSERVATION_AREA_FEATURE = "Within conservation area"
# Named "Tree canopy" (not "Street tree") because the underlying density unions # Named "Tree canopy" (not "Street tree") because the underlying density unions
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a # Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
@ -77,23 +83,42 @@ _AREA_COLUMNS = [
"% Mixed", "% Mixed",
"% White", "% White",
"% Other", "% Other",
# Crime # Crime — average annual recorded incident count (incidents/yr), 7-year and
"Anti-social behaviour (avg/yr)", # 2-year windows. These are the filterable crime features; the per-incident
"Violence and sexual offences (avg/yr)", # records live in a separate side table the server loads directly (it bypasses
"Criminal damage and arson (avg/yr)", # the merge).
"Burglary (avg/yr)", "Anti-social behaviour (/yr, 7y)",
"Vehicle crime (avg/yr)", "Anti-social behaviour (/yr, 2y)",
"Robbery (avg/yr)", "Violence and sexual offences (/yr, 7y)",
"Other theft (avg/yr)", "Violence and sexual offences (/yr, 2y)",
"Shoplifting (avg/yr)", "Criminal damage and arson (/yr, 7y)",
"Drugs (avg/yr)", "Criminal damage and arson (/yr, 2y)",
"Possession of weapons (avg/yr)", "Burglary (/yr, 7y)",
"Public order (avg/yr)", "Burglary (/yr, 2y)",
"Bicycle theft (avg/yr)", "Vehicle crime (/yr, 7y)",
"Theft from the person (avg/yr)", "Vehicle crime (/yr, 2y)",
"Other crime (avg/yr)", "Robbery (/yr, 7y)",
"Serious crime (avg/yr)", "Robbery (/yr, 2y)",
"Minor crime (avg/yr)", "Other theft (/yr, 7y)",
"Other theft (/yr, 2y)",
"Shoplifting (/yr, 7y)",
"Shoplifting (/yr, 2y)",
"Drugs (/yr, 7y)",
"Drugs (/yr, 2y)",
"Possession of weapons (/yr, 7y)",
"Possession of weapons (/yr, 2y)",
"Public order (/yr, 7y)",
"Public order (/yr, 2y)",
"Bicycle theft (/yr, 7y)",
"Bicycle theft (/yr, 2y)",
"Theft from the person (/yr, 7y)",
"Theft from the person (/yr, 2y)",
"Other crime (/yr, 7y)",
"Other crime (/yr, 2y)",
"Serious crime (/yr, 7y)",
"Serious crime (/yr, 2y)",
"Minor crime (/yr, 7y)",
"Minor crime (/yr, 2y)",
# Amenities # Amenities
"Number of restaurants within 2km", "Number of restaurants within 2km",
"Number of grocery shops and supermarkets within 2km", "Number of grocery shops and supermarkets within 2km",
@ -189,8 +214,6 @@ _FINAL_RENAME_COLUMNS = {
"outstanding_primary_catchments": "Outstanding primary school catchments", "outstanding_primary_catchments": "Outstanding primary school catchments",
"outstanding_secondary_catchments": "Outstanding secondary school catchments", "outstanding_secondary_catchments": "Outstanding secondary school catchments",
"max_download_speed": "Max available download speed (Mbps)", "max_download_speed": "Max available download speed (Mbps)",
"serious_crime_avg_yr": "Serious crime (avg/yr)",
"minor_crime_avg_yr": "Minor crime (avg/yr)",
"mean_monthly_rent": "Estimated monthly rent", "mean_monthly_rent": "Estimated monthly rent",
"floor_height": "Interior height (m)", "floor_height": "Interior height (m)",
"was_council_house": "Former council house", "was_council_house": "Former council house",
@ -822,78 +845,6 @@ def _validate_property_postcodes(df: pl.DataFrame) -> None:
) )
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
"""Return the supported postcode universe with geography join keys."""
return (
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
.filter(pl.col("doterm").is_null())
.select(
pl.col("pcds").alias("postcode"),
"lat",
pl.col("long").alias("lon"),
"ctry25cd",
pl.col("lsoa21cd").alias("lsoa21"),
pl.col("oa21cd").alias("oa21"),
pl.col("pcon24cd").alias("pcon"),
)
.drop_nulls(["postcode"])
.unique(["postcode"])
)
def _remap_terminated_postcodes(
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
) -> pl.LazyFrame:
return (
wide.join(
postcode_mapping,
left_on="postcode",
right_on="old_postcode",
how="left",
)
.with_columns(
pl.coalesce("new_postcode", "postcode").alias("postcode"),
)
.drop("new_postcode")
)
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
"""Keep one row per (postcode, address) — the most-recent transaction.
The terminated-postcode remap can map two distinct postcodes onto one active
successor, collapsing the same physical address onto a single
(postcode, address) key with conflicting sale records. Keep the row with the
latest date_of_transfer so the headline price/date reflect the most recent
transaction; genuinely distinct addresses are untouched.
The dedup key coalesces the price-paid address with the EPC address: EPC-only
dwellings (never sold) have a null pp_address, so keying on pp_address alone
would collapse EVERY EPC-only dwelling in a postcode onto one
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
address is unique within its postcode (the EPC frame is deduped on
address+postcode upstream), so the coalesced key keeps them distinct while
leaving sold-property dedup unchanged pp_address wins the coalesce whenever
a sale exists.
"""
return (
wide.with_columns(
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
)
.sort("date_of_transfer", descending=True, nulls_last=True)
.unique(
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
)
.drop("_dedupe_address")
)
def _filter_to_active_english_postcodes(
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
) -> pl.LazyFrame:
return wide.join(active_postcodes, on="postcode", how="semi")
def _join_area_side_tables( def _join_area_side_tables(
base: pl.LazyFrame, base: pl.LazyFrame,
*, *,
@ -923,21 +874,15 @@ def _join_area_side_tables(
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age. # joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
base = base.join(tenure, on="lsoa21", how="left") base = base.join(tenure, on="lsoa21", how="left")
# Crime is counted spatially per postcode (incidents within 50m of the # Crime is counted spatially per postcode (incidents within the boundary
# postcode boundary), so it joins on postcode rather than LSOA. crime_spatial # buffer), so it joins on postcode rather than LSOA. crime_spatial writes
# precomputes the Serious/Minor headline rollups as the mean of the by-year # average-annual-count columns ("{type} (/yr, 7y|2y)"), including the
# rollup bars; read those straight through (renamed to the internal columns # Serious/Minor rollups (the exact sum of their components); all pass straight
# _finalize_merged_columns expects) rather than re-summing the per-type # through to display/filtering. A postcode absent from the crime table keeps
# avg/yr columns — summing divides each type by its OWN years-present and # null values via the left join (no fabricated zero). The per-incident records
# overstates the rollup when types differ in coverage. A postcode absent from # are a separate side table the server loads directly, so it is not joined
# the crime table keeps null rollups via the left join (no fabricated zero); # here.
# the per-type avg/yr columns pass through unchanged for display. base = base.join(crime, on="postcode", how="left")
base = base.join(crime, on="postcode", how="left").rename(
{
"Serious crime (avg/yr)": "serious_crime_avg_yr",
"Minor crime (avg/yr)": "minor_crime_avg_yr",
}
)
base = base.join(median_age, on="lsoa21", how="left") base = base.join(median_age, on="lsoa21", how="left")
base = base.join(election, on="pcon", how="left") base = base.join(election, on="pcon", how="left")
@ -2386,27 +2331,17 @@ def _build(
) )
_validate_lad_source_coverage(iod_path, rental_prices_path) _validate_lad_source_coverage(iod_path, rental_prices_path)
wide = pl.scan_parquet(epc_pp_path).filter( # The dwelling universe — floor filter, terminated-postcode remap,
pl.col("total_floor_area").is_null() # collapse-dedupe, restrict to active English postcodes — is shared with
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2) # price estimation so estimates line up 1:1 with these rows. See
) # pipeline.transform.property_base.
wide = build_property_base(epc_pp_path, arcgis_path)
# Remap terminated postcodes to nearest active successor before filtering to
# the supported active-English postcode universe. Historical properties from
# terminated English postcodes are retained under their successor postcode.
postcode_mapping = build_postcode_mapping(arcgis_path)
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
# The remap can collapse two terminated postcodes onto one active successor,
# duplicating a physical address's (postcode, pp_address) key; keep only the
# most-recent transaction per address before the per-postcode joins.
wide = _dedupe_collapsed_properties(wide)
arcgis_raw = pl.scan_parquet(arcgis_path) arcgis_raw = pl.scan_parquet(arcgis_path)
arcgis = _active_english_postcode_area(arcgis_raw) arcgis = _active_english_postcode_area(arcgis_raw)
active_postcodes = arcgis.select("postcode").unique() active_postcodes = arcgis.select("postcode").unique()
active_postcode_count = ( active_postcode_count = (
active_postcodes.select(pl.len()).collect(engine="streaming").item() active_postcodes.select(pl.len()).collect(engine="streaming").item()
) )
wide = _filter_to_active_english_postcodes(wide, active_postcodes)
if listed_buildings_path is not None: if listed_buildings_path is not None:
active_postcodes_for_listed = ( active_postcodes_for_listed = (
@ -2542,37 +2477,10 @@ def _build(
how="left", how="left",
) )
# Derive property_type: prefer EPC data, fall back to price-paid. # Derive property_type (EPC preferred, price-paid fallback, built_form for
# For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer detail. # houses). Shared with price_inputs so the estimate uses the same type; see
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in( # property_base.property_type_expr.
["NO DATA!", "Not Recorded"] wide = wide.with_columns(property_type_expr().alias("property_type"))
)
has_epc = pl.col("epc_property_type").is_not_null()
is_house = pl.col("epc_property_type") == "House"
wide = wide.with_columns(
pl.when(has_epc & is_house & ~bad_built_form)
.then(pl.col("built_form"))
.when(has_epc & is_house)
.then(pl.col("pp_property_type"))
.when(has_epc)
.then(pl.col("epc_property_type"))
.otherwise(pl.col("pp_property_type"))
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
# collapse terrace sub-types, and fold rare types into "Other"
.replace(
{
"Flat": "Flats/Maisonettes",
"Maisonette": "Flats/Maisonettes",
"End-Terrace": "Terraced",
"Mid-Terrace": "Terraced",
"Enclosed End-Terrace": "Terraced",
"Enclosed Mid-Terrace": "Terraced",
"Bungalow": "Other",
"Park home": "Other",
}
)
.alias("property_type")
)
wide = wide.with_columns( wide = wide.with_columns(
pl.when(pl.col("duration") == "U") pl.when(pl.col("duration") == "U")

View file

@ -79,7 +79,9 @@ The output of `process_oa` is `list[(postcode, polygon)]` — the per-OA fragmen
**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 30005000 m² building blocks) before this change. **Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 30005000 m² building blocks) before this change.
**GeoJSON output** (`output.py:write_district_geojson`): Two passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties. **Greenspace subtraction is connectivity-preserving** (`greenspace.py:subtract_greenspace`): park/water polygons are subtracted from each postcode, but greenspace that *crosses* a postcode (a river, a strip of parkland, a golf course through a village) would otherwise split it into scattered pieces. When the subtraction disconnects a postcode, `_reconnect_split` re-adds the narrowest removed necks — a morphological closing (`_RECONNECT_BRIDGE_M`, 25 m) clipped to the original postcode footprint — so parts ≤ ~50 m apart stay joined by a thin bridge of the postcode's own land (no address moves); genuinely wide barriers stay subtracted and the postcode legitimately splits.
**GeoJSON output** (`output.py:write_district_geojson`): three passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 **de-fragments** the partition (`_eliminate_small_detached_parts`): a detached part that is *both* small in absolute terms (< `_ELIM_ABS_MAX_M2`, 3000 m²) *and* a minor fraction (< `_ELIM_FRAC_MAX`, 15%) of its postcode is absorbed into the neighbouring postcode it shares the most boundary with — the classic GIS *eliminate*. This removes the Voronoi/INSPIRE/seam *scatter* that left ~1/3 of postcodes non-contiguous, while a genuine bisection (two substantial parts split by a river/railway) keeps both parts. The land is **reassigned**, never dropped, so the output stays a gapless partition and coverage is conserved; the largest part of every postcode is always retained, so no active postcode is dropped (a tiny neighbour-less sliver in removed greenspace is dropped, a larger isolated patch is kept). Pass 3 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
## Memory architecture ## Memory architecture
@ -107,6 +109,7 @@ Key design choices:
2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs 2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries 3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
4. **A postcode split across an OA seam keeps all its substantial parts**`merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped 4. **A postcode split across an OA seam keeps all its substantial parts**`merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
5. **Postcodes are contiguous unless genuinely split** — most non-contiguity is *scatter* (a unit drawn as many disconnected specks) from point-Voronoi over sparse/interleaved UPRNs, greenspace cutting across a unit, and overlap/seam slivers. Connectivity-preserving greenspace subtraction + the `_eliminate_small_detached_parts` de-fragmentation pass absorb that scatter into neighbours (coverage-conserving), cutting the share of multi-part postcodes roughly in half (~30% → ~14% measured on the worst rural/coastal districts) without dropping any postcode or leaving coverage gaps. Genuine bisections (river/railway/major road, or a detached part above the absolute+fraction thresholds) are preserved.
## Module structure ## Module structure

View file

@ -7,7 +7,7 @@ from shapely import make_valid, wkb
from shapely.geometry import MultiPolygon, Polygon from shapely.geometry import MultiPolygon, Polygon
from shapely.strtree import STRtree from shapely.strtree import STRtree
from .geometry import safe_difference, safe_union from .geometry import _SNAP_GRID, _poly_valid, safe_difference, safe_intersection, safe_union
def load_greenspace(path: Path) -> tuple[STRtree, list]: def load_greenspace(path: Path) -> tuple[STRtree, list]:
@ -36,6 +36,42 @@ def load_greenspace(path: Path) -> tuple[STRtree, list]:
MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed
# Greenspace that merely trims the edge of a postcode is fine, but greenspace
# that CROSSES it (a river, a strip of parkland, a golf course running through a
# village) splits the postcode into a MultiPolygon -- one the map then draws as
# several disconnected pieces. When subtraction disconnects a postcode, re-add
# the postcode's OWN removed land along the narrowest necks (a morphological
# closing clipped to the original footprint) so the parts stay joined by a thin
# bridge. Parts left more than ~2x this width apart (a genuinely wide barrier)
# stay split. Because the bridge is the postcode's own land, no address moves and
# only a thin sliver of green is kept back.
_RECONNECT_BRIDGE_M = 25.0
def _reconnect_split(
result: Polygon | MultiPolygon, postcode_geom: Polygon | MultiPolygon
) -> Polygon | MultiPolygon:
"""Re-join postcode parts that greenspace subtraction pulled apart by re-adding
the narrow removed necks (within the original postcode), leaving wide barriers
intact."""
if result.geom_type != "MultiPolygon":
return result
closed = result.buffer(_RECONNECT_BRIDGE_M).buffer(-_RECONNECT_BRIDGE_M)
if not closed.is_valid:
closed = make_valid(closed)
# The closing material that lies inside the original postcode but outside the
# subtraction result == the thin green necks linking the parts. The exact
# overlay path can leave line/point debris (coincident edges) that is
# zero-area but NOT is_empty; `_poly_valid` strips it to polygons only, so the
# is_empty guard works and the union can never return a GeometryCollection
# (which `to_wgs84_geojson_multi` would silently truncate to a single part).
bridges = _poly_valid(
safe_difference(safe_intersection(closed, postcode_geom), result), _SNAP_GRID
)
if bridges.is_empty:
return result
return _poly_valid(safe_union([result, bridges]), _SNAP_GRID)
def subtract_greenspace( def subtract_greenspace(
postcode_geom: Polygon | MultiPolygon, postcode_geom: Polygon | MultiPolygon,
@ -48,6 +84,10 @@ def subtract_greenspace(
of intersecting greenspace from the postcode polygon. If subtraction of intersecting greenspace from the postcode polygon. If subtraction
would remove >90% of the area, keeps the original (the postcode would remove >90% of the area, keeps the original (the postcode
genuinely covers that land, e.g. churchyards, riverside addresses). genuinely covers that land, e.g. churchyards, riverside addresses).
If the subtraction disconnects the postcode (greenspace crossing it),
:func:`_reconnect_split` re-adds the narrowest removed necks so the postcode
stays a single piece rather than shipping as scattered fragments.
""" """
candidate_idxs = tree.query(postcode_geom) candidate_idxs = tree.query(postcode_geom)
if len(candidate_idxs) == 0: if len(candidate_idxs) == 0:
@ -74,4 +114,4 @@ def subtract_greenspace(
if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION): if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION):
return postcode_geom return postcode_geom
return result return _reconnect_split(result, postcode_geom)

View file

@ -1,4 +1,5 @@
import json import json
import math
import shutil import shutil
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
@ -383,6 +384,164 @@ def _resolve_overlaps(
return [(pc, out[i]) for i, (pc, _) in enumerate(items)] return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
# A detached (non-largest) part of a postcode is absorbed into the neighbouring
# postcode it shares the most boundary with, when the part is BOTH small in
# absolute terms AND a minor fraction of its postcode. This removes the
# Voronoi/INSPIRE/overlap-seam SCATTER that otherwise left ~1/3 of postcodes
# non-contiguous (a single unit drawn as many disconnected specks), while keeping
# genuine splits: a postcode bisected by a river/railway into two SUBSTANTIAL
# parts has both parts above these thresholds and is left alone. The land is
# REASSIGNED to an adjacent postcode wherever a neighbour exists, so coverage is
# conserved and the output stays a partition; only a tiny neighbour-less sliver
# (< _ELIM_DROP_BELOW_M2, snap/overlap debris floating in removed greenspace) is
# dropped, and a larger isolated part (a genuine detached hamlet) is kept. The
# largest part of every postcode is always retained, so no postcode is dropped.
_ELIM_ABS_MAX_M2 = 3000.0 # absolute size below which a minor part reads as scatter
_ELIM_FRAC_MAX = 0.15 # ...and only when it is < this fraction of the postcode
_ELIM_DROP_BELOW_M2 = 200.0 # a neighbour-less sliver this small is snap debris -> drop
# WGS84-degree distances at UK latitudes (1e-6 deg ~ 0.11 m at ~53N)
_ELIM_SHARED_EPS_DEG = 5e-7 # ~0.05 m: thin probe whose overlap area ~ shared-edge length
# Candidate-gather + nearest-neighbour fallback radius, in degrees so it needs no
# per-part reprojection. The metric reach is mildly anisotropic (~2.2 m N-S,
# ~1.3-1.4 m E-W at England's latitudes); this only bounds the rare gapped-seam
# fallback (the dominant border-sharing path is unaffected), so the approximation
# is harmless. Gather and accept use the SAME radius, so there is no dead band.
_ELIM_NEAREST_MAX_DEG = 2e-5
_M_PER_DEG_LAT = 111_320.0
_ELIM_ITERATIONS = 2
def _approx_area_m2(geom_deg) -> float:
"""Metric area of a WGS84 polygon via a latitude-scaled planar approximation.
Accurate to a few percent at the few-hundred-to-few-thousand m^2 scale these
thresholds work at, and far cheaper than a per-part CRS transform over the
full ~1.8M postcodes.
"""
area_deg2 = geom_deg.area
if area_deg2 == 0.0:
return 0.0
centroid = geom_deg.centroid
if centroid.is_empty:
lat = (geom_deg.bounds[1] + geom_deg.bounds[3]) / 2
else:
lat = centroid.y
return area_deg2 * _M_PER_DEG_LAT * _M_PER_DEG_LAT * math.cos(math.radians(lat))
def _geom_parts(geom):
return list(geom.geoms) if geom.geom_type == "MultiPolygon" else [geom]
def _eliminate_small_detached_parts(
items: list[tuple[str, Polygon | MultiPolygon]],
) -> list[tuple[str, Polygon | MultiPolygon]]:
"""De-fragment the partition: absorb small detached parts into the neighbour
they border most (see the module note above).
Runs on the de-overlapped WGS84 geometries as the last shaping step. The
largest part of every postcode is always retained, so no active postcode is
dropped; parts are moved between postcodes (or, for tiny neighbour-less
slivers, dropped), so total covered area is conserved to within rounding.
"""
geoms: dict[str, Polygon | MultiPolygon] = {pc: g for pc, g in items}
for _ in range(_ELIM_ITERATIONS):
recs: list[tuple[str, Polygon, float]] = []
total: dict[str, float] = defaultdict(float)
largest: dict[str, float] = defaultdict(float)
for pc, geom in geoms.items():
for part in _geom_parts(geom):
if part.is_empty:
continue
area = _approx_area_m2(part)
recs.append((pc, part, area))
total[pc] += area
if area > largest[pc]:
largest[pc] = area
tree = STRtree([part for _, part, _ in recs])
assign: dict[int, str | None] = {}
for i, (pc, part, area) in enumerate(recs):
if area >= largest[pc]:
continue # the largest part always stays -> a postcode never vanishes
if not (area < _ELIM_ABS_MAX_M2 and area < _ELIM_FRAC_MAX * total[pc]):
continue # substantial or major-fraction part = genuine split, keep
best_pc = None
best_score = 0.0
nearest_pc = None
nearest_d = float("inf")
# Gather candidates out to the nearest-fallback radius (not just the
# smaller border-probe radius), so a gapped seam in the
# (border, nearest] band can actually be reassigned rather than dropped.
for j in tree.query(part.buffer(_ELIM_NEAREST_MAX_DEG)):
if j == i:
continue
other_pc, other_geom, _ = recs[j]
if other_pc == pc:
continue
try:
score = part.buffer(_ELIM_SHARED_EPS_DEG).intersection(other_geom).area
except GEOSException:
score = 0.0
# Ties broken by the lexicographically smaller postcode so the
# result is independent of STRtree traversal order (matches the
# stable tie-break in _resolve_overlaps; keeps output byte-stable
# across shapely/GEOS upgrades for content-hash caching).
if score > best_score or (
score == best_score and best_pc is not None and other_pc < best_pc
):
best_score = score
best_pc = other_pc
dist = part.distance(other_geom)
if dist < nearest_d or (
dist == nearest_d
and nearest_pc is not None
and other_pc < nearest_pc
):
nearest_d = dist
nearest_pc = other_pc
if best_pc is not None and best_score > 0:
assign[i] = best_pc # shares a border -> absorb into that neighbour
elif nearest_pc is not None and nearest_d <= _ELIM_NEAREST_MAX_DEG:
assign[i] = nearest_pc # gapped seam -> nearest neighbour
elif area < _ELIM_DROP_BELOW_M2:
assign[i] = None # neighbour-less snap/overlap sliver -> drop
# else: keep with its own postcode (a genuine isolated patch)
if not assign:
break
new_parts: dict[str, list] = defaultdict(list)
for i, (pc, part, _) in enumerate(recs):
if i in assign:
target = assign[i]
if target is None:
continue
new_parts[target].append(part)
else:
new_parts[pc].append(part)
rebuilt: dict[str, Polygon | MultiPolygon] = {}
for pc, parts in new_parts.items():
if len(parts) == 1:
rebuilt[pc] = parts[0]
else:
merged = safe_union(parts, grid=_OUTPUT_PRECISION_DEG)
if not merged.is_empty:
rebuilt[pc] = merged
# Carry forward any postcode that contributed no parts to recs (e.g. an
# all-empty input geometry, which the part loop skips): never drop a
# postcode just because reassignment fired elsewhere in this pass.
for pc, geom in geoms.items():
if pc not in rebuilt:
rebuilt[pc] = geom
geoms = rebuilt
return list(geoms.items())
def _round_coords(coords, ndigits=6): def _round_coords(coords, ndigits=6):
if coords and isinstance(coords[0], (int, float)): if coords and isinstance(coords[0], (int, float)):
return [round(coords[0], ndigits), round(coords[1], ndigits)] return [round(coords[0], ndigits), round(coords[1], ndigits)]
@ -501,6 +660,11 @@ def write_district_geojson(
# Remove overlap strips so the output is a clean partition. # Remove overlap strips so the output is a clean partition.
converted = _resolve_overlaps(converted) converted = _resolve_overlaps(converted)
# De-fragment: absorb small detached parts (Voronoi/INSPIRE/seam scatter) into
# the neighbour they border most, so postcodes stop shipping as many
# disconnected specks. Coverage-preserving; runs on the final partition.
converted = _eliminate_small_detached_parts(converted)
by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list) by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
for pc, geom in converted: for pc, geom in converted:
parts = pc.split() parts = pc.split()

View file

@ -4,6 +4,7 @@ Each test targets a specific bug or edge case identified during code review.
""" """
import json import json
import math
import numpy as np import numpy as np
import polars as pl import polars as pl
@ -21,6 +22,8 @@ from .inspire import build_inspire_index
from .oa_boundaries import parse_gpkg_geometry from .oa_boundaries import parse_gpkg_geometry
from .greenspace import subtract_greenspace from .greenspace import subtract_greenspace
from .output import ( from .output import (
_approx_area_m2,
_eliminate_small_detached_parts,
_fill_holes, _fill_holes,
merge_fragments, merge_fragments,
to_wgs84_geojson, to_wgs84_geojson,
@ -1830,3 +1833,231 @@ class TestFragmentsCache:
cache.write_text("c") cache.write_text("c")
# arcgis is optional/absent — it cannot have invalidated the cache. # arcgis is optional/absent — it cannot have invalidated the cache.
assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True
# ---------------------------------------------------------------------------
# De-fragmentation: small detached parts are absorbed into their best neighbour
# ---------------------------------------------------------------------------
# WGS84 helper: build a box from METRE offsets around a fixed England anchor, so
# the eliminate pass (which works in WGS84 degrees and measures area with a
# latitude-scaled approximation) sees realistic coordinates and areas.
_ANCHOR_LON, _ANCHOR_LAT = -0.1, 51.5
_M_PER_DEG_LAT = 111_320.0
_M_PER_DEG_LON = 111_320.0 * math.cos(math.radians(_ANCHOR_LAT))
def _mbox(x0, y0, x1, y1):
return box(
_ANCHOR_LON + x0 / _M_PER_DEG_LON,
_ANCHOR_LAT + y0 / _M_PER_DEG_LAT,
_ANCHOR_LON + x1 / _M_PER_DEG_LON,
_ANCHOR_LAT + y1 / _M_PER_DEG_LAT,
)
def _as_dict(items):
return dict(items)
class TestEliminateSmallDetachedParts:
"""A small detached part should be absorbed into the postcode it borders most,
de-fragmenting the unit while conserving coverage and never dropping a
postcode. Genuine large splits must survive."""
def test_small_island_absorbed_by_surrounding_neighbour(self):
# A: a big main blob (left) plus a small 30x30m island sitting INSIDE B's
# territory; B fills the middle/right with a hole where the island is.
main_a = _mbox(0, 0, 100, 200) # 20000 m²
island = _mbox(150, 90, 180, 120) # 900 m², surrounded by B
a = MultiPolygon([main_a, island])
b = _mbox(100, 0, 300, 200).difference(island) # hole around the island
before = unary_union([a, b]).area
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
assert "A" in out and "B" in out, "no postcode may be dropped"
assert out["A"].geom_type == "Polygon", "A's island should be absorbed away"
# The island area moves to B (the surrounding postcode); coverage is kept.
assert out["B"].contains(island.representative_point())
after = unary_union([out["A"], out["B"]]).area
assert after == pytest.approx(before, rel=1e-6), "coverage must be conserved"
def test_genuine_large_split_is_kept(self):
# Two substantial parts (both 40000 m²) far apart — a real bisection, not
# scatter. Neither is below the absolute/fraction thresholds, so both stay.
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(400, 0, 600, 200)])
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
assert out["A"].geom_type == "MultiPolygon"
assert len(out["A"].geoms) == 2
def test_midsize_minor_part_kept_when_above_abs_threshold(self):
# A big main plus a 10000 m² detached part: above the 3000 m² absolute
# threshold, so it is a genuine secondary piece and must NOT be eliminated,
# even though it is a small fraction of the postcode.
a = MultiPolygon([_mbox(0, 0, 300, 300), _mbox(1000, 0, 1100, 100)])
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
assert out["A"].geom_type == "MultiPolygon"
def test_tiny_neighbourless_sliver_is_dropped(self):
# A main blob plus a 10x10m (100 m²) sliver far from anything: below the
# drop threshold with no neighbour to absorb it -> dropped as snap debris.
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(1000, 1000, 1010, 1010)])
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
assert "A" in out, "the postcode itself must survive"
assert out["A"].geom_type == "Polygon"
def test_largest_part_always_retained_no_postcode_dropped(self):
# Even a postcode that is ENTIRELY a tiny sliver keeps its (largest) part —
# active postcodes must never be dropped (validate_outputs is zero-tolerance).
tiny = _mbox(500, 500, 505, 505) # 25 m², the whole postcode
big = _mbox(0, 0, 300, 300)
out = _as_dict(
_eliminate_small_detached_parts([("TINY", tiny), ("BIG", big)])
)
assert "TINY" in out and not out["TINY"].is_empty
assert "BIG" in out
def test_no_overlaps_introduced(self):
# Absorbing parts must keep the output a partition (no double-coverage).
main_a = _mbox(0, 0, 100, 200)
island = _mbox(150, 90, 180, 120)
a = MultiPolygon([main_a, island])
b = _mbox(100, 0, 300, 200).difference(island)
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
overlap = out["A"].intersection(out["B"]).area
assert overlap < (1.0 / (_M_PER_DEG_LAT * _M_PER_DEG_LON)), "no overlap"
def test_empty_postcode_not_dropped_when_reassignment_fires(self):
# Regression: an empty-geometry postcode must survive even when a
# reassignment fires elsewhere in the same pass (the rebuild used to drop
# any key absent from `recs`). No active postcode may ever be dropped.
main_a = _mbox(0, 0, 100, 200)
island = _mbox(150, 90, 180, 120) # B absorbs this -> reassignment fires
a = MultiPolygon([main_a, island])
b = _mbox(100, 0, 300, 200).difference(island)
empty = Polygon() # degenerate input that contributes no parts
out = _as_dict(
_eliminate_small_detached_parts([("A", a), ("B", b), ("EMPTY", empty)])
)
assert "EMPTY" in out, "an empty-geometry postcode must not be dropped"
assert "A" in out and "B" in out
def test_tiebreak_is_deterministic_smaller_postcode_wins(self):
# An island bordered by TWO postcodes with EQUAL shared edges must go to
# the lexicographically smaller postcode, independent of STRtree order.
a_main = _mbox(0, 0, 100, 100) # A's main, far from the island
island = _mbox(300, 50, 320, 70) # 400 m², A's detached part
a = MultiPolygon([a_main, island])
bbb = _mbox(250, 0, 300, 120) # borders island's left edge
ccc = _mbox(320, 0, 370, 120) # borders island's right edge (equal)
out = _as_dict(
_eliminate_small_detached_parts([("A", a), ("BBB", bbb), ("CCC", ccc)])
)
pt = island.representative_point()
assert out["BBB"].contains(pt), "tie must go to the smaller postcode string"
assert not out["CCC"].contains(pt)
assert out["A"].geom_type == "Polygon"
def test_gapped_sliver_within_nearest_radius_is_reassigned_not_dropped(self):
# A 300 m² sliver 1.2 m from its only neighbour N — beyond the border probe
# but inside the nearest-fallback radius. Before the gather buffer was
# widened to the accept radius, the old (smaller) gather buffer never
# returned N, so the sliver fell through to the drop branch. Now it is
# reassigned to N, conserving coverage.
x_main = _mbox(0, 0, 100, 100) # far from the sliver
x_sliver = _mbox(278.8, 50, 298.8, 65) # 300 m², right edge at x=298.8
x = MultiPolygon([x_main, x_sliver])
n = _mbox(300, 0, 400, 200) # left edge at x=300 -> 1.2 m gap
before = unary_union([x, n]).area
out = _as_dict(_eliminate_small_detached_parts([("X", x), ("N", n)]))
assert out["X"].geom_type == "Polygon", "sliver should leave X"
assert out["N"].contains(x_sliver.representative_point()), "absorbed into N"
after = unary_union([out["X"], out["N"]]).area
assert after == pytest.approx(before, rel=1e-6), "coverage conserved, not dropped"
def test_approx_area_matches_real_metric_area(self):
# The latitude-scaled area approximation should be within a few percent of
# the true projected area for a building-scale polygon.
import pyproj
from shapely.ops import transform as transform_geometry
poly = _mbox(0, 0, 50, 60) # ~3000 m²
to_bng = pyproj.Transformer.from_crs(
"EPSG:4326", "EPSG:27700", always_xy=True
)
true_m2 = transform_geometry(to_bng.transform, poly).area
assert _approx_area_m2(poly) == pytest.approx(true_m2, rel=0.02)
# ---------------------------------------------------------------------------
# Greenspace subtraction is connectivity-preserving
# ---------------------------------------------------------------------------
class TestGreenspaceReconnect:
"""Greenspace that CROSSES a postcode must not shatter it: a narrow strip is
bridged back (postcode stays one piece), a wide barrier is left subtracted
(postcode genuinely splits)."""
def test_narrow_strip_reconnects_postcode(self):
from shapely.strtree import STRtree
postcode = box(0, 0, 200, 100)
strip = box(95, 0, 105, 100) # 10 m wide green strip crossing the postcode
# Sanity: a plain difference WOULD split it into two parts.
assert postcode.difference(strip).geom_type == "MultiPolygon"
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
assert result.geom_type == "Polygon", "narrow strip should be bridged"
assert result.is_valid
def test_wide_barrier_keeps_postcode_split(self):
from shapely.strtree import STRtree
postcode = box(0, 0, 200, 100)
barrier = box(60, 0, 130, 100) # 70 m wide — beyond 2x the bridge width
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
assert result.geom_type == "MultiPolygon", "wide barrier should stay split"
assert len(result.geoms) == 2
def test_edge_greenspace_still_trimmed(self):
# Greenspace on the edge (not crossing) is trimmed as before; reconnect is
# a no-op because the result stays a single piece.
from shapely.strtree import STRtree
postcode = box(0, 0, 100, 100) # 10000 m²
park = box(60, 0, 100, 100) # 4000 m² on the right edge
result = subtract_greenspace(postcode, STRtree([park]), [park])
assert result.geom_type == "Polygon"
assert result.area == pytest.approx(6000, rel=0.01)
def test_result_is_always_polygonal(self):
# Regression: _reconnect_split must never return a GeometryCollection
# (line/point debris) — downstream to_wgs84_geojson_multi would truncate a
# GC to a single part, silently dropping substantial pieces. Sweep many
# strip widths/offsets (incl. coincident-edge-prone integer geometries).
from shapely.strtree import STRtree
postcode = box(0, 0, 200, 120)
for w in (2, 5, 8, 10, 25, 49, 50, 51, 60, 90):
for x0 in (40, 70, 95, 100, 130):
strip = box(x0, -10, x0 + w, 130)
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
assert result.geom_type in ("Polygon", "MultiPolygon"), (
f"w={w} x0={x0} produced {result.geom_type}"
)
assert result.is_valid and not result.is_empty
def test_wide_barrier_preserves_all_substantial_parts(self):
# The motivating case for the GC fix: a wide barrier genuinely splits the
# postcode; ALL substantial parts must survive (not be truncated to one).
from shapely.strtree import STRtree
postcode = box(0, 0, 300, 100)
barrier = box(120, 0, 200, 100) # 80 m wide -> beyond 2x bridge
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
assert result.geom_type == "MultiPolygon"
areas = sorted(p.area for p in result.geoms)
assert areas == pytest.approx([10000, 12000], rel=0.01) # both banks kept

View file

@ -1,4 +1,4 @@
"""Augment properties.parquet with estimated current prices. """Estimate current prices for the merged properties, as a standalone artifact.
For properties with a known prior sale, applies the repeat-sales price index For properties with a known prior sale, applies the repeat-sales price index
to adjust the last known price to the current date, then blends with kNN to adjust the last known price to the current date, then blends with kNN
@ -6,8 +6,13 @@ estimates from nearby recently-sold properties. Includes:
- Capping extreme index adjustments - Capping extreme index adjustments
- kNN spatial blending - kNN spatial blending
Modifies properties.parquet in-place. Temporarily joins postcode.parquet Reads the slim price_inputs.parquet (built by property_base, independently of
for lat/lon needed by kNN, then drops those columns before writing. merge's area features) plus postcode.parquet for the lat/lon kNN needs, and
writes a slim price_estimates.parquet of just the natural key (Postcode +
coalesced address) and "Estimated current price" / "Est. price per sqm".
join_price_estimates.py joins those two columns back onto properties.parquet.
Because the inputs do not depend on merge's area columns, adding such a column
does not invalidate this step.
""" """
import argparse import argparse
@ -26,12 +31,27 @@ from pipeline.transform.price_estimation.knn import (
from pipeline.transform.price_estimation.utils import ( from pipeline.transform.price_estimation.utils import (
CURRENT_FRAC_YEAR, CURRENT_FRAC_YEAR,
CURRENT_YEAR, CURRENT_YEAR,
ESTIMATE_COLUMNS,
JOIN_KEYS,
MAX_LOG_ADJUSTMENT, MAX_LOG_ADJUSTMENT,
interpolate_log_index, interpolate_log_index,
join_address_expr,
sector_expr, sector_expr,
type_group_expr, type_group_expr,
) )
# Columns estimate reads from price_inputs.parquet. The two address columns are
# only carried to build the natural join key (Postcode + coalesced address).
INPUT_COLUMNS = [
"Postcode",
"Property type",
"Total floor area (sqm)",
"Last known price",
"Date of last transaction",
"Address per Property Register",
"Address per EPC",
]
MAX_KNN_TO_INDEX_RATIO = 2.0 MAX_KNN_TO_INDEX_RATIO = 2.0
MIN_KNN_TO_INDEX_RATIO = 0.5 MIN_KNN_TO_INDEX_RATIO = 0.5
# Cap the final estimate at this multiple of the last known price as a guard # Cap the final estimate at this multiple of the last known price as a guard
@ -161,13 +181,14 @@ def guarded_blend_estimates(
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Augment properties.parquet with estimated current prices" description="Estimate current prices for the merged properties"
) )
parser.add_argument( parser.add_argument(
"--properties", "--input",
type=Path, type=Path,
required=True, required=True,
help="Path to properties.parquet (modified in-place)", help="Path to price_inputs.parquet (slim per-dwelling inputs from "
"property_base)",
) )
parser.add_argument( parser.add_argument(
"--postcodes", "--postcodes",
@ -178,22 +199,23 @@ def main():
parser.add_argument( parser.add_argument(
"--index", type=Path, required=True, help="Path to price_index.parquet" "--index", type=Path, required=True, help="Path to price_index.parquet"
) )
parser.add_argument(
"--output",
type=Path,
required=True,
help="Output price_estimates.parquet (natural key + estimate columns)",
)
args = parser.parse_args() args = parser.parse_args()
print("Loading properties.parquet...") print("Loading price inputs (projection)...")
df = pl.read_parquet(args.properties) df = pl.read_parquet(args.input, columns=INPUT_COLUMNS)
print(f" {len(df):,} rows, {len(df.columns)} columns") print(f" {len(df):,} rows, {len(INPUT_COLUMNS)} input columns")
# Join lat/lon from postcode.parquet for kNN spatial queries # Join lat/lon from postcode.parquet for kNN spatial queries
postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon") postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon")
df = df.join(postcodes, on="Postcode", how="left") df = df.join(postcodes, on="Postcode", how="left")
print(f" Joined lat/lon from {len(postcodes):,} postcodes") print(f" Joined lat/lon from {len(postcodes):,} postcodes")
# Drop existing estimated columns if re-running
for col in ["Estimated current price", "Est. price per sqm"]:
if col in df.columns:
df = df.drop(col)
# Derive helper columns # Derive helper columns
df = df.with_columns( df = df.with_columns(
sector_expr().alias("_sector"), sector_expr().alias("_sector"),
@ -355,16 +377,15 @@ def main():
.alias("Est. price per sqm"), .alias("Est. price per sqm"),
) )
# Drop all temporary columns and joined lat/lon (those belong in postcode.parquet) # Emit only the natural join key and the two estimate columns.
temp_cols = [c for c in df.columns if c.startswith("_") or c.startswith("log_idx_")] # join_price_estimates.py joins these back onto properties.parquet.
df = df.drop(temp_cols).drop("lat", "lon") result = df.with_columns(join_address_expr()).select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
df.write_parquet(args.properties) result.write_parquet(args.output)
size_mb = args.properties.stat().st_size / (1024 * 1024) size_mb = args.output.stat().st_size / (1024 * 1024)
print(f"\nWrote {args.properties} ({size_mb:.1f} MB)") print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
print( n_priced = result.filter(pl.col("Estimated current price").is_not_null()).height
f" {len(df):,} rows, {len(df.columns)} columns (including 'Estimated current price')" print(f" {len(result):,} rows, {n_priced:,} with an estimated current price")
)
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -16,6 +16,26 @@ LATEST_COMPLETE_YEAR = CURRENT_YEAR - 1
_today = date.today() _today = date.today()
CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12 CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12
# The two columns price estimation contributes to properties.parquet, kept here
# so both the producer (estimate) and the joiner (join_price_estimates) agree.
ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
# Natural join key from estimates back onto properties: postcode plus the
# coalesced register/EPC address. This is unique and non-null on the deduped
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
# estimates 1:1 onto properties regardless of row order — estimates are computed
# from a separate price_inputs.parquet, so a positional key would not line up.
JOIN_ADDRESS = "_join_address"
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
def join_address_expr() -> pl.Expr:
"""The coalesced address half of the natural key, aliased to JOIN_ADDRESS."""
return pl.coalesce("Address per Property Register", "Address per EPC").alias(
JOIN_ADDRESS
)
# Cap on log(index_ratio) to prevent wild estimates from thin sectors # Cap on log(index_ratio) to prevent wild estimates from thin sectors
MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change
TERRACE_TYPES = [ TERRACE_TYPES = [

View file

@ -0,0 +1,217 @@
"""Shared property base: the dwelling universe before any area enrichment.
This is the single source of truth for *which* dwellings exist and their
intrinsic, source-level attributes (price, floor area, type, addresses). Both
``merge`` (which enriches it with postcode/LSOA-keyed area features to build
properties.parquet) and price estimation (which only needs the intrinsic
columns) start from exactly these rows, so estimates computed here line up 1:1
with the final properties by the natural key ``(Postcode, coalesced address)``.
Living in its own module is what lets price estimation be *cached* across
merge changes: ``price_inputs.parquet`` depends only on epc_pp + arcgis + this
file, so adding an area column to merge.py does not invalidate it and the
expensive index/kNN steps are skipped.
"""
import argparse
from pathlib import Path
import polars as pl
from pipeline.utils.postcode_mapping import build_postcode_mapping
MIN_FLOOR_AREA_M2 = 10
# Columns price estimation reads, with the final (properties.parquet) names so
# index.py/estimate.py and the join all speak the same schema. The two address
# columns form the natural join key (Postcode + their coalesce).
PRICE_INPUT_SELECT = [
pl.col("postcode").alias("Postcode"),
pl.col("total_floor_area").alias("Total floor area (sqm)"),
pl.col("latest_price").alias("Last known price"),
pl.col("date_of_transfer").alias("Date of last transaction"),
"historical_prices",
pl.col("pp_address").alias("Address per Property Register"),
pl.col("epc_address").alias("Address per EPC"),
]
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
"""Return the supported postcode universe with geography join keys."""
return (
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
.filter(pl.col("doterm").is_null())
.select(
pl.col("pcds").alias("postcode"),
"lat",
pl.col("long").alias("lon"),
"ctry25cd",
pl.col("lsoa21cd").alias("lsoa21"),
pl.col("oa21cd").alias("oa21"),
pl.col("pcon24cd").alias("pcon"),
)
.drop_nulls(["postcode"])
.unique(["postcode"])
)
def _remap_terminated_postcodes(
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
) -> pl.LazyFrame:
return (
wide.join(
postcode_mapping,
left_on="postcode",
right_on="old_postcode",
how="left",
)
.with_columns(
pl.coalesce("new_postcode", "postcode").alias("postcode"),
)
.drop("new_postcode")
)
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
"""Keep one row per (postcode, address) — the most-recent transaction.
The terminated-postcode remap can map two distinct postcodes onto one active
successor, collapsing the same physical address onto a single
(postcode, address) key with conflicting sale records. Keep the row with the
latest date_of_transfer so the headline price/date reflect the most recent
transaction; genuinely distinct addresses are untouched.
The dedup key coalesces the price-paid address with the EPC address: EPC-only
dwellings (never sold) have a null pp_address, so keying on pp_address alone
would collapse EVERY EPC-only dwelling in a postcode onto one
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
address is unique within its postcode (the EPC frame is deduped on
address+postcode upstream), so the coalesced key keeps them distinct while
leaving sold-property dedup unchanged pp_address wins the coalesce whenever
a sale exists.
"""
return (
wide.with_columns(
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
)
.sort("date_of_transfer", descending=True, nulls_last=True)
.unique(
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
)
.drop("_dedupe_address")
)
def _filter_to_active_english_postcodes(
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
) -> pl.LazyFrame:
return wide.join(active_postcodes, on="postcode", how="semi")
def property_type_expr() -> pl.Expr:
"""Unaliased property-type expression: prefer EPC, fall back to price-paid.
For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer
detail. Depends only on intrinsic base columns (epc_property_type,
pp_property_type, built_form), so merge and price_inputs derive the same
value. Callers alias it ("property_type" in merge, "Property type" in
price_inputs).
"""
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
["NO DATA!", "Not Recorded"]
)
has_epc = pl.col("epc_property_type").is_not_null()
is_house = pl.col("epc_property_type") == "House"
return (
pl.when(has_epc & is_house & ~bad_built_form)
.then(pl.col("built_form"))
.when(has_epc & is_house)
.then(pl.col("pp_property_type"))
.when(has_epc)
.then(pl.col("epc_property_type"))
.otherwise(pl.col("pp_property_type"))
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
# collapse terrace sub-types, and fold rare types into "Other"
.replace(
{
"Flat": "Flats/Maisonettes",
"Maisonette": "Flats/Maisonettes",
"End-Terrace": "Terraced",
"Mid-Terrace": "Terraced",
"Enclosed End-Terrace": "Terraced",
"Enclosed Mid-Terrace": "Terraced",
"Bungalow": "Other",
"Park home": "Other",
}
)
)
def build_postcode_centroids(arcgis_path: Path) -> pl.LazyFrame:
"""One row per active-English postcode with its lat/lon, from arcgis.
This is the lat/lon source price estimation needs (index sector centroids,
kNN). It is the same per-postcode lat/lon merge writes into postcode.parquet
(both come from arcgis), but built straight from arcgis so the index/estimate
steps do not depend on the merge output adding an area column to merge
therefore does not invalidate the expensive price index/kNN.
"""
return _active_english_postcode_area(pl.scan_parquet(arcgis_path)).select(
pl.col("postcode").alias("Postcode"), "lat", "lon"
)
def build_property_base(epc_pp_path: Path, arcgis_path: Path) -> pl.LazyFrame:
"""The deduped, active-English dwelling universe from epc_pp + arcgis.
Floor filter -> terminated-postcode remap -> collapse-dedupe -> restrict to
the active English postcode universe. Returns a LazyFrame with the original
epc_pp column names; merge enriches it, the CLI projects it to price_inputs.
"""
wide = pl.scan_parquet(epc_pp_path).filter(
pl.col("total_floor_area").is_null()
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
)
postcode_mapping = build_postcode_mapping(arcgis_path)
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
wide = _dedupe_collapsed_properties(wide)
active_postcodes = (
_active_english_postcode_area(pl.scan_parquet(arcgis_path))
.select("postcode")
.unique()
)
return _filter_to_active_english_postcodes(wide, active_postcodes)
def main():
parser = argparse.ArgumentParser(
description="Write the slim price-estimation inputs from epc_pp + arcgis"
)
parser.add_argument("--epc-pp", type=Path, required=True)
parser.add_argument("--arcgis", type=Path, required=True)
parser.add_argument(
"--output", type=Path, required=True, help="price_inputs.parquet output"
)
parser.add_argument(
"--centroids",
type=Path,
required=True,
help="postcode_centroids.parquet output (Postcode, lat, lon)",
)
args = parser.parse_args()
base = build_property_base(args.epc_pp, args.arcgis)
price_inputs = base.with_columns(
property_type_expr().alias("Property type")
).select(*PRICE_INPUT_SELECT, "Property type")
price_inputs.sink_parquet(args.output)
n = pl.scan_parquet(args.output).select(pl.len()).collect().item()
print(f"Wrote {args.output} ({args.output.stat().st_size / 1e6:.1f} MB), {n:,} dwellings")
build_postcode_centroids(args.arcgis).sink_parquet(args.centroids)
n_pc = pl.scan_parquet(args.centroids).select(pl.len()).collect().item()
print(f"Wrote {args.centroids} ({args.centroids.stat().st_size / 1e6:.1f} MB), {n_pc:,} postcodes")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,81 @@
import polars as pl
from pipeline.transform.area_crime_averages import (
NATIONAL_AREA,
SCOPE_NATIONAL,
SCOPE_OUTCODE,
SCOPE_SECTOR,
compute_area_crime_averages,
)
_BURGLARY = "Burglary (/yr, 7y)"
_ROBBERY = "Robbery (/yr, 7y)"
def _postcodes() -> pl.LazyFrame:
return pl.LazyFrame(
{
"Postcode": ["E14 2DG", "E14 2AB", "E14 9XY", "M1 1AE", "E14 2ZZ"],
# E14 9XY has no usable crime data; E14 2AB lacks robbery; E14 2ZZ has
# crime but (below) no properties, so it must not weight any average.
_BURGLARY: [10.0, 20.0, None, 5.0, 100.0],
_ROBBERY: [2.0, None, None, 1.0, 50.0],
# An unrelated column proves only the crime columns are averaged.
"Median age": [40.0, 41.0, 42.0, 30.0, 99.0],
}
)
def _properties() -> pl.LazyFrame:
# Property rows per postcode become the weights (3 / 1 / 2 / 4). E14 2ZZ has
# none, so it is excluded entirely.
postcodes = ["E14 2DG"] * 3 + ["E14 2AB"] + ["E14 9XY"] * 2 + ["M1 1AE"] * 4
return pl.LazyFrame({"Postcode": postcodes})
def _row(df: pl.DataFrame, scope: str, area: str) -> dict:
matched = df.filter((pl.col("scope") == scope) & (pl.col("area") == area))
assert matched.height == 1, f"expected one {scope} row for {area!r}"
return matched.to_dicts()[0]
def test_property_weighted_means_skip_nulls():
result = compute_area_crime_averages(_postcodes(), _properties())
national = _row(result, SCOPE_NATIONAL, NATIONAL_AREA)
# Burglary: (10*3 + 20*1 + 5*4) / (3+1+4) = 70/8; E14 9XY null dilutes nothing.
assert national[_BURGLARY] == 8.75
# Robbery: (2*3 + 1*4) / (3+4) = 10/7; both null postcodes are excluded from
# the numerator AND the denominator.
assert national[_ROBBERY] == pl.Series([10.0 / 7.0]).cast(pl.Float32).item()
outcode = _row(result, SCOPE_OUTCODE, "E14")
assert outcode[_BURGLARY] == 12.5 # (10*3 + 20*1) / 4
assert outcode[_ROBBERY] == 2.0 # only E14 2DG has robbery (2 * 3 / 3)
def test_sector_aggregation_and_all_null_rows_dropped():
result = compute_area_crime_averages(_postcodes(), _properties())
sector = _row(result, SCOPE_SECTOR, "E14 2")
assert sector[_BURGLARY] == 12.5
assert sector[_ROBBERY] == 2.0
# E14 9XY has properties but no crime data at all, so its sector "E14 9" is
# all-null and must be dropped rather than reported as a known area.
assert result.filter(pl.col("area") == "E14 9").height == 0
def test_postcodes_without_properties_are_excluded():
result = compute_area_crime_averages(_postcodes(), _properties())
# E14 2ZZ carries crime values but no properties; including it would pull the
# E14 outcode burglary mean toward its 100.0. It must contribute nothing.
outcode = _row(result, SCOPE_OUTCODE, "E14")
assert outcode[_BURGLARY] == 12.5
def test_only_crime_columns_are_emitted():
result = compute_area_crime_averages(_postcodes(), _properties())
assert set(result.columns) == {"scope", "area", _BURGLARY, _ROBBERY}
assert result.schema[_BURGLARY] == pl.Float32

View file

@ -1,13 +1,10 @@
import json import json
import numpy as np
import polars as pl import polars as pl
import pytest import pytest
import shapely
from pyproj import Transformer from pyproj import Transformer
from pipeline.transform.crime_spatial import transform_crime_spatial from pipeline.transform.crime_spatial import transform_crime_spatial
from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True) _TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
@ -16,6 +13,10 @@ _CSV_HEADER = (
"LSOA code,LSOA name,Crime type,Last outcome category,Context" "LSOA code,LSOA name,Crime type,Last outcome category,Context"
) )
# Average-annual-count crime column name for a window (the filterable feature).
def _raw(t: str, window: str = "7y") -> str:
return f"{t} (/yr, {window})"
def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]: def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]:
lon, lat = _TO_WGS84.transform(x, y) lon, lat = _TO_WGS84.transform(x, y)
@ -39,12 +40,12 @@ def _write_boundaries(units_dir, features_by_district: dict[str, list[dict]]) ->
(units_dir / f"{district}.geojson").write_text(json.dumps(collection)) (units_dir / f"{district}.geojson").write_text(json.dumps(collection))
def _crime_row(month: str, x, y, crime_type: str) -> str: def _crime_row(month: str, x, y, crime_type: str, location="On or near X", outcome="U") -> str:
if x is None or y is None: if x is None or y is None:
lon, lat = "", "" lon, lat = "", ""
else: else:
lon, lat = _bng_to_wgs84(x, y) lon, lat = _bng_to_wgs84(x, y)
return f",{month},F,F,{lon},{lat},On or near X,E01000001,L,{crime_type},U," return f",{month},F,F,{lon},{lat},{location},E01000001,L,{crime_type},{outcome},"
def _write_month( def _write_month(
@ -59,10 +60,22 @@ def _write_month(
def _run(tmp_path, crime, units, **kwargs): def _run(tmp_path, crime, units, **kwargs):
output = tmp_path / "crime_by_postcode.parquet" """Run the transform and return (crime, by_year, records) DataFrames.
The crime table carries the average-annual-count columns ("{type} (/yr, …)"),
i.e. the raw, absolute number of recorded incidents per year.
"""
crime_out = tmp_path / "crime_by_postcode.parquet"
by_year = tmp_path / "crime_by_postcode_by_year.parquet" by_year = tmp_path / "crime_by_postcode_by_year.parquet"
transform_crime_spatial(crime, units, output, by_year, buffer_m=50.0, **kwargs) records = tmp_path / "crime_records.parquet"
return pl.read_parquet(output), pl.read_parquet(by_year) transform_crime_spatial(
crime, units, crime_out, by_year, records, buffer_m=50.0, **kwargs
)
return (
pl.read_parquet(crime_out),
pl.read_parquet(by_year),
pl.read_parquet(records),
)
def test_buffer_overlap_counts_for_each_postcode(tmp_path): def test_buffer_overlap_counts_for_each_postcode(tmp_path):
@ -95,17 +108,74 @@ def test_buffer_overlap_counts_for_each_postcode(tmp_path):
], ],
) )
avg_df, _ = _run(tmp_path, crime, units) raw_df, _, _ = _run(tmp_path, crime, units)
rows = {r["postcode"]: r for r in avg_df.to_dicts()} rows = {r["postcode"]: r for r in raw_df.to_dicts()}
# Single covered month -> pooled rate x12. # Single covered month -> pooled raw rate x12.
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == 12.0 assert rows["AB1 1AA"][_raw("Burglary")] == 12.0
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == 12.0 assert rows["AB1 1AB"][_raw("Burglary")] == 12.0
assert rows["AB1 1AA"]["Robbery (avg/yr)"] == 0.0 assert rows["AB1 1AA"][_raw("Robbery")] == 0.0
# Only the 49m robbery counts for C; the 51m one and the blank row do not. # Only the 49m robbery counts for C; the 51m one and the blank row do not.
assert rows["AB1 1AC"]["Robbery (avg/yr)"] == 12.0 assert rows["AB1 1AC"][_raw("Robbery")] == 12.0
assert rows["AB1 1AC"]["Burglary (avg/yr)"] == 0.0 assert rows["AB1 1AC"][_raw("Burglary")] == 0.0
# Anti-social behaviour had no coordinate -> nobody gets it. # Anti-social behaviour had no coordinate -> nobody gets it.
assert all(r["Anti-social behaviour (avg/yr)"] == 0.0 for r in rows.values()) assert all(r[_raw("Anti-social behaviour")] == 0.0 for r in rows.values())
def test_counts_are_not_area_normalised(tmp_path):
# Three postcodes of very different footprint, each with exactly one incident
# in its buffer. The raw count must be 12/yr for ALL of them: area
# normalisation has been removed, so footprint no longer changes the number.
units = tmp_path / "units"
_write_boundaries(
units,
{
"AB1": [
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20
_square_feature("AB1 1AC", 5000, 5000, 5040, 5040), # 40x40
]
},
)
crime = tmp_path / "crime"
_write_month(
crime,
"2024-01",
[
_crime_row("2024-01", 1005, 1005, "Burglary"),
_crime_row("2024-01", 3005, 3010, "Burglary"),
_crime_row("2024-01", 5020, 5020, "Burglary"),
],
)
raw_df, _, _ = _run(tmp_path, crime, units, min_bar_months=1)
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
for pc in ("AB1 1AA", "AB1 1AB", "AB1 1AC"):
assert rows[pc][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
def test_windows_pool_only_recent_years(tmp_path):
# 2-year window vs 7-year window. An incident in the latest year sits in both
# windows; one 6 years back sits only in the 7-year window.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
)
crime = tmp_path / "crime"
# 12 covered months in 2019 (1 burglary), 12 in 2025 (1 burglary). Latest =
# 2025: 7y window = 2019..2025 (both), 2y window = 2024..2025 (only 2025).
for month in range(1, 13):
ym19 = f"2019-{month:02d}"
ym25 = f"2025-{month:02d}"
_write_month(crime, ym19, [_crime_row(ym19, 1005, 1005, "Burglary")] if month == 1 else [])
_write_month(crime, ym25, [_crime_row(ym25, 1005, 1005, "Burglary")] if month == 1 else [])
raw_df, _, _ = _run(tmp_path, crime, units)
row = raw_df.row(0, named=True)
# 7y: 2 incidents over 24 covered months -> 1/yr.
assert row[_raw("Burglary", "7y")] == pytest.approx(1.0, abs=0.05)
# 2y: 1 incident over 12 covered months -> 1/yr (the 2019 one is excluded).
assert row[_raw("Burglary", "2y")] == pytest.approx(1.0, abs=0.05)
def test_by_year_annualises_and_rolls_up(tmp_path): def test_by_year_annualises_and_rolls_up(tmp_path):
@ -115,7 +185,6 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
) )
crime = tmp_path / "crime" crime = tmp_path / "crime"
# Point at the centre of AB1 1AA, well inside its buffer.
_write_month( _write_month(
crime, crime,
"2023-01", "2023-01",
@ -134,7 +203,7 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
], ],
) )
_, by_year_df = _run(tmp_path, crime, units, min_bar_months=1) _, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
assert by_year_df.height == 1 assert by_year_df.height == 1
cols = set(by_year_df.columns) cols = set(by_year_df.columns)
assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols
@ -150,77 +219,14 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
# 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12). # 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12).
assert serious[2023] == 24.0 assert serious[2023] == 24.0
assert serious[2024] == 12.0 assert serious[2024] == 12.0
# Coverage calendar: both years published, with their month counts.
coverage = {c["year"]: c["months"] for c in row["covered_years"]} coverage = {c["year"]: c["months"] for c in row["covered_years"]}
assert coverage == {2023: 1, 2024: 2} assert coverage == {2023: 1, 2024: 2}
def test_area_normalisation_divides_out_buffered_catchment(tmp_path): def test_raw_is_pooled_rate_over_covered_months(tmp_path):
# Three postcodes of increasing footprint, each with exactly one incident in # Uneven month coverage: 2023 has 1 month (2 incidents), 2024 has 2 months
# its buffer. Normalisation rescales by median_catchment / buffered_area, so # (2 incidents). The raw figure is the POOLED annualised rate over all covered
# the smallest scores highest and the median-sized one is unchanged -- i.e. # months: 4 incidents / 3 months * 12 = 16/yr.
# the metric is a density. Dividing by the *buffered* catchment (not the raw
# polygon) means the fixed buffer-ring floor keeps the spread gentle, so the
# tiniest postcode is not blown up out of proportion.
units = tmp_path / "units"
_write_boundaries(
units,
{
"AB1": [
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20 (median)
_square_feature("AB1 1AC", 5000, 5000, 5020, 5020), # 20x20
]
},
)
crime = tmp_path / "crime"
_write_month(
crime,
"2024-01",
[
_crime_row("2024-01", 1005, 1005, "Burglary"),
_crime_row("2024-01", 3005, 3010, "Burglary"),
_crime_row("2024-01", 5010, 5010, "Burglary"),
],
)
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
# Re-derive the expected values from the same buffered catchment areas: each
# postcode is 12/yr before normalisation, then x (median_buf / buffered_area).
postcodes, polygons = load_postcode_polygons(units)
buf_area = {
pc: float(shapely.area(shapely.buffer(poly, 50.0, quad_segs=8)))
for pc, poly in zip(postcodes, polygons)
}
median_buf = float(np.median(list(buf_area.values())))
expected = {pc: 12.0 * median_buf / buf_area[pc] for pc in buf_area}
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
for pc, exp in expected.items():
assert rows[pc]["Burglary (avg/yr)"] == pytest.approx(exp, abs=0.1)
# Median catchment unchanged; ordering is by inverse buffered area, but the
# buffer-ring floor keeps the spread far below the ~4x raw-area ratio.
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
small = rows["AB1 1AA"]["Burglary (avg/yr)"]
big = rows["AB1 1AC"]["Burglary (avg/yr)"]
assert small > 12.0 > big
assert small / big < 1.5
# by-year series carries the same normalisation.
small_row = by_year_df.filter(pl.col("postcode") == "AB1 1AA").row(0, named=True)
assert small_row["Burglary (by year)"] == [
{"year": 2024, "count": pytest.approx(expected["AB1 1AA"], abs=0.1)}
]
def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
# Uneven month coverage across years: 2023 has 1 month (2 incidents),
# 2024 has 2 months (2 incidents). The headline is the POOLED annualised
# rate over all covered months: 4 incidents / 3 months * 12 = 16/yr -- not
# the old mean-of-bars (24+12)/2 = 18, which over-weighted thin years.
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]} units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@ -238,10 +244,9 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")]) _write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
_write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")]) _write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")])
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1) raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
avg = avg_df.row(0, named=True) assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(16.0, abs=0.05)
assert avg["Burglary (avg/yr)"] == pytest.approx(16.0, abs=0.05)
# Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6). # Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6).
row = by_year_df.row(0, named=True) row = by_year_df.row(0, named=True)
@ -251,8 +256,7 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
def test_sporadic_type_is_not_inflated_by_years_present(tmp_path): def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
# A single robbery in a 24-covered-month window must read as ~0.5/yr (the # A single robbery in a 24-covered-month window must read as ~0.5/yr (the
# long-run pooled rate), NOT 12/yr (the old years-with-incidents mean that # long-run pooled rate), NOT 12/yr.
# inflated sporadic categories by up to ~15x).
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]} units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@ -266,14 +270,10 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")] rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")]
_write_month(crime, f"{year}-{month:02d}", rows) _write_month(crime, f"{year}-{month:02d}", rows)
avg_df, by_year_df = _run(tmp_path, crime, units) raw_df, by_year_df, _ = _run(tmp_path, crime, units)
avg = avg_df.row(0, named=True)
# 1 incident over 24 covered months -> 0.5/yr. # 1 incident over 24 covered months -> 0.5/yr.
assert avg["Robbery (avg/yr)"] == pytest.approx(0.5, abs=0.05) assert raw_df.row(0, named=True)[_raw("Robbery")] == pytest.approx(0.5, abs=0.05)
# The by-year bar still shows the 2023 incident annualised over 12 covered
# months (1/yr); 2024 is covered with zero robberies -> no bar, but the
# year IS in the coverage list so consumers may render it as a true zero.
row = by_year_df.row(0, named=True) row = by_year_df.row(0, named=True)
bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]} bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]}
assert bars == {2023: pytest.approx(1.0, abs=0.05)} assert bars == {2023: pytest.approx(1.0, abs=0.05)}
@ -283,9 +283,8 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
def test_force_gap_years_are_excluded_not_zeroed(tmp_path): def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
# Two postcodes policed by different forces. force-a publishes 2023+2024; # Two postcodes policed by different forces. force-a publishes 2023+2024;
# force-b publishes only 2023 (a 2024 gap, like Greater Manchester). The # force-b publishes only 2023 (a 2024 gap). The b-postcode's raw figure must
# b-postcode's headline must pool over force-b's 12 covered months only, # pool over force-b's 12 covered months only.
# and its by-year series must NOT contain a 2024 bar or coverage entry.
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
units, units,
@ -299,25 +298,21 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
for month in range(1, 13): for month in range(1, 13):
ym23 = f"2023-{month:02d}" ym23 = f"2023-{month:02d}"
ym24 = f"2024-{month:02d}" ym24 = f"2024-{month:02d}"
# force-a covers AB1 in both years; one burglary per month in 2024.
_write_month(crime, ym23, [], force="force-a") _write_month(crime, ym23, [], force="force-a")
_write_month( _write_month(
crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a" crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a"
) )
# force-b covers CD1 in 2023 only: one burglary per month.
_write_month( _write_month(
crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b" crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b"
) )
avg_df, by_year_df = _run(tmp_path, crime, units) raw_df, by_year_df, _ = _run(tmp_path, crime, units)
rows = {r["postcode"]: r for r in avg_df.to_dicts()} rows = {r["postcode"]: r for r in raw_df.to_dicts()}
# force-a postcode: 12 burglaries over 24 covered months -> 6/yr. # force-a postcode: 12 burglaries over 24 covered months -> 6/yr.
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05) assert rows["AB1 1AA"][_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
# force-b postcode: 12 burglaries over 12 covered months -> 12/yr. Under # force-b postcode: 12 burglaries over 12 covered months -> 12/yr.
# the old global calendar this would have been diluted to 6/yr by the assert rows["CD1 1AA"][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
# uncovered 2024.
assert rows["CD1 1AA"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()} by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()}
b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]} b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]}
@ -328,59 +323,10 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
assert a_coverage == {2023: 12, 2024: 12} assert a_coverage == {2023: 12, 2024: 12}
def test_residue_incidents_in_uncovered_years_are_excluded(tmp_path):
# force-b stops publishing after 2023, but a force-a file contains a 2024
# incident that falls inside the b-postcode's buffer (cross-border residue,
# the Greater Manchester pattern). That incident must not produce a 2024
# bar for the b-postcode, nor leak into its pooled headline.
units = tmp_path / "units"
_write_boundaries(
units,
{
"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)],
"CD1": [_square_feature("CD1 1AA", 9000, 9000, 9010, 9010)],
},
)
crime = tmp_path / "crime"
for month in range(1, 13):
ym23 = f"2023-{month:02d}"
ym24 = f"2024-{month:02d}"
_write_month(crime, ym23, [], force="force-a")
# b's own 2023 incidents establish force-b as its home force.
_write_month(
crime,
ym23,
[_crime_row(ym23, 9005, 9005, "Burglary")] if month <= 6 else [],
force="force-b",
)
# 2024: only force-a publishes; one of its incidents lands in CD1 1AA.
_write_month(
crime,
ym24,
[_crime_row(ym24, 9005, 9005, "Burglary")] if month == 1 else [],
force="force-a",
)
avg_df, by_year_df = _run(tmp_path, crime, units)
b_row = avg_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
# Pooled over force-b's 12 covered months (2023): 6 incidents -> 6/yr.
# The residue 2024 incident is excluded (force-b published 0 months in 2024).
assert b_row["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
b_by = by_year_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
bars = {p["year"]: p["count"] for p in b_by["Burglary (by year)"]}
assert set(bars) == {2023}
coverage = {c["year"]: c["months"] for c in b_by["covered_years"]}
assert coverage == {2023: 12}
def test_partial_years_below_min_bar_months_get_no_bar(tmp_path): def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
# 2023 fully covered; 2024 has only 2 published months. With the default # 2023 fully covered; 2024 has only 2 published months. With the default
# 6-month minimum, 2024 must produce neither a bar (annualising x6 charts # 6-month minimum, 2024 must produce no bar -- but its incidents and months
# noise) nor a coverage entry -- but its incidents and months still count # still count toward the pooled raw figure.
# toward the pooled headline.
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]} units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@ -394,12 +340,10 @@ def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
ym = f"2024-{month:02d}" ym = f"2024-{month:02d}"
_write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")]) _write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")])
avg_df, by_year_df = _run(tmp_path, crime, units) raw_df, by_year_df, _ = _run(tmp_path, crime, units)
# Pooled: 14 incidents over 14 covered months -> 12/yr. # Pooled: 14 incidents over 14 covered months -> 12/yr.
assert avg_df.row(0, named=True)["Burglary (avg/yr)"] == pytest.approx( assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
12.0, abs=0.05
)
row = by_year_df.row(0, named=True) row = by_year_df.row(0, named=True)
bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]} bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]}
assert set(bars) == {2023} assert set(bars) == {2023}
@ -425,52 +369,119 @@ def test_by_year_output_is_dense_with_coverage(tmp_path):
crime = tmp_path / "crime" crime = tmp_path / "crime"
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")]) _write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1) raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
assert by_year_df.height == 2 assert by_year_df.height == 2
quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True) quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
assert quiet["Burglary (by year)"] is None assert quiet["Burglary (by year)"] is None
assert [c["year"] for c in quiet["covered_years"]] == [2024] assert [c["year"] for c in quiet["covered_years"]] == [2024]
# And the headline for the quiet postcode is a genuine 0, not null. # The raw figure for the covered, crime-free postcode is a genuine 0, not null.
quiet_avg = avg_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True) quiet_raw = raw_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
assert quiet_avg["Burglary (avg/yr)"] == 0.0 assert quiet_raw[_raw("Burglary")] == 0.0
def test_serious_rollup_avg_yr_equals_sum_of_components(tmp_path): def test_serious_rollup_equals_sum_of_components(tmp_path):
# Burglary only in 2014, Robbery only in 2024 (one incident each, 2 covered # Burglary only in 2023, Robbery only in 2024 (one incident each, 2 covered
# months total). Components pool over the same covered window (each # months total, both inside the 7-year window). Components pool over the same
# 1 x 12 / 2 = 6/yr) and the rollup equals their sum. # covered window (each 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]} units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
) )
crime = tmp_path / "crime" crime = tmp_path / "crime"
_write_month(crime, "2014-01", [_crime_row("2014-01", 1005, 1005, "Burglary")]) _write_month(crime, "2023-01", [_crime_row("2023-01", 1005, 1005, "Burglary")])
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")]) _write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")])
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1) raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
avg = avg_df.row(0, named=True) row = raw_df.row(0, named=True)
assert avg["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05) assert row[_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
assert avg["Robbery (avg/yr)"] == pytest.approx(6.0, abs=0.05) assert row[_raw("Robbery")] == pytest.approx(6.0, abs=0.05)
# Rollup == sum of its component (avg/yr) columns. assert row[_raw("Serious crime")] == pytest.approx(12.0, abs=0.05)
assert avg["Serious crime (avg/yr)"] == pytest.approx(12.0, abs=0.05) assert row[_raw("Serious crime")] == pytest.approx(
assert avg["Serious crime (avg/yr)"] == pytest.approx( row[_raw("Burglary")] + row[_raw("Robbery")], abs=0.05
avg["Burglary (avg/yr)"] + avg["Robbery (avg/yr)"], abs=0.05
) )
# The by-year rollup series remains the per-year sum of the component bars.
serious_bars = { serious_bars = {
p["year"]: p["count"] p["year"]: p["count"]
for p in by_year_df.row(0, named=True)["Serious crime (by year)"] for p in by_year_df.row(0, named=True)["Serious crime (by year)"]
} }
assert serious_bars == { assert serious_bars == {
2014: pytest.approx(12.0, abs=0.05), 2023: pytest.approx(12.0, abs=0.05),
2024: pytest.approx(12.0, abs=0.05), 2024: pytest.approx(12.0, abs=0.05),
} }
def test_records_capture_each_counted_incident(tmp_path):
# Each (incident, postcode) match within the records window becomes a record
# row, carrying month/type/location/outcome/coords. A boundary incident
# counted for two postcodes appears once per postcode.
units = tmp_path / "units"
_write_boundaries(
units,
{
"AB1": [
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010),
_square_feature("AB1 1AB", 1080, 1000, 1090, 1010),
]
},
)
crime = tmp_path / "crime"
_write_month(
crime,
"2024-03",
[
# In the buffer overlap -> recorded for both postcodes.
_crime_row("2024-03", 1045, 1005, "Burglary", location="On or near High St", outcome="Under investigation"),
# Only in AB1 1AA's buffer; null outcome (police.uk leaves ASB blank).
_crime_row("2024-03", 1005, 1005, "Anti-social behaviour", location="On or near Mill Ln", outcome=""),
],
)
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
assert set(records_df.columns) == {
"postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"
}
# Sorted by postcode.
assert records_df["postcode"].is_sorted()
# Burglary appears for BOTH postcodes (boundary multiplicity); ASB only for AA.
by_pc = records_df.group_by("postcode").agg(pl.col("crime_type").sort())
counts = {r["postcode"]: r["crime_type"] for r in by_pc.to_dicts()}
assert counts["AB1 1AA"] == ["Anti-social behaviour", "Burglary"]
assert counts["AB1 1AB"] == ["Burglary"]
# month_index = year*12 + (month-1) for 2024-03.
assert set(records_df["month_index"].to_list()) == {2024 * 12 + 2}
# Null outcome round-trips as null, not the string "".
asb = records_df.filter(pl.col("crime_type") == "Anti-social behaviour").row(0, named=True)
assert asb["outcome"] is None
assert asb["location"] == "On or near Mill Ln"
def test_records_window_aligns_to_the_headline_calendar_window(tmp_path):
# Records must cover exactly the longest (7y) headline window, which is
# calendar-year based. With a mid-year latest month (2025-06) the 7y window
# is calendar years 2019..2025, so an incident in 2018-09 -- which the
# headline excludes -- must also be excluded from the records, even though a
# naive rolling 84-month span (ending 2025-06) would wrongly include it. The
# first month of the earliest window year (2019-01) is kept.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
)
crime = tmp_path / "crime"
_write_month(crime, "2018-09", [_crime_row("2018-09", 1005, 1005, "Burglary")])
_write_month(crime, "2019-01", [_crime_row("2019-01", 1005, 1005, "Burglary")])
_write_month(crime, "2025-06", [_crime_row("2025-06", 1005, 1005, "Burglary")])
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
# 2018-09 (year*12+8) is in the rolling 84-month span but NOT the 7y calendar
# window, so it is excluded; 2019-01 and 2025-06 are kept.
assert set(records_df["month_index"].to_list()) == {2019 * 12 + 0, 2025 * 12 + 5}
def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys): def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
units = tmp_path / "units" units = tmp_path / "units"
_write_boundaries( _write_boundaries(
@ -487,11 +498,10 @@ def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
], ],
) )
avg_df, _ = _run(tmp_path, crime, units) raw_df, _, _ = _run(tmp_path, crime, units)
columns = avg_df.columns columns = raw_df.columns
# The unknown type is dropped (no column for it) but a warning is emitted. assert _raw("Cyber fraud") not in columns
assert "Cyber fraud (avg/yr)" not in columns assert _raw("Burglary") in columns
assert "Burglary (avg/yr)" in columns
err = capsys.readouterr().err err = capsys.readouterr().err
assert "Cyber fraud" in err assert "Cyber fraud" in err
assert "WARNING" in err assert "WARNING" in err
@ -515,11 +525,11 @@ def test_legacy_crime_types_are_mapped(tmp_path):
], ],
) )
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1) raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
row = avg_df.to_dicts()[0] row = raw_df.to_dicts()[0]
# Single postcode -> area-norm factor 1.0; single covered month -> x12. # Single covered month (relative to a 2013-latest window) -> x12.
assert row["Violence and sexual offences (avg/yr)"] == 12.0 assert row[_raw("Violence and sexual offences")] == 12.0
assert row["Public order (avg/yr)"] == 12.0 assert row[_raw("Public order")] == 12.0
by_year_row = by_year_df.row(0, named=True) by_year_row = by_year_df.row(0, named=True)
assert by_year_row["Violence and sexual offences (by year)"] == [ assert by_year_row["Violence and sexual offences (by year)"] == [

View file

@ -0,0 +1,136 @@
"""Tests for joining slim price estimates back onto properties.parquet.
estimate.py emits (Postcode, coalesced address, estimate columns) and
join_estimates attaches them by that natural key. These tests pin the
properties that make the key safe: it maps estimates onto the right rows
regardless of order (a shuffled estimates frame is the worst case), it is
idempotent, and it refuses a partial/foreign estimates file rather than
silently nulling prices.
"""
from pathlib import Path
import polars as pl
import pytest
from pipeline.transform.join_price_estimates import join_estimates
from pipeline.transform.price_estimation.utils import (
ESTIMATE_COLUMNS,
JOIN_ADDRESS,
JOIN_KEYS,
join_address_expr,
)
N = 200
def _write_merged(path: Path) -> pl.DataFrame:
"""properties.parquet with the natural-key columns, a sentinel order column,
and no estimates. Half the rows are sale-addressed, half EPC-only, so the
coalesce in the key is exercised; every coalesced address is unique."""
df = pl.DataFrame(
{
"Postcode": [f"AA{i % 7} {i % 9}AA" for i in range(N)],
"Address per Property Register": [
f"reg-{i}" if i % 2 == 0 else None for i in range(N)
],
"Address per EPC": [f"epc-{i}" if i % 2 == 1 else None for i in range(N)],
"order": list(range(N)),
"junk": [f"x{i}" for i in range(N)],
}
)
df.write_parquet(path)
return df
def _write_estimates(path: Path, merged_path: Path, *, shuffle: bool = True) -> None:
"""Estimates keyed by the natural key, derived from the merged file the way
estimate.py does. Estimate = order * 1000 so each row is checkable. Shuffled
by default to prove order-independence."""
est = (
pl.read_parquet(merged_path)
.with_columns(join_address_expr())
.with_columns(
(pl.col("order") * 1000).cast(pl.Float64).alias("Estimated current price"),
(pl.col("order") * 10).cast(pl.Int32).alias("Est. price per sqm"),
)
.select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
)
if shuffle:
est = est.sample(fraction=1.0, shuffle=True, seed=7)
est.write_parquet(path)
def test_join_attaches_estimates_to_the_right_rows(tmp_path: Path):
props = tmp_path / "properties.parquet"
estimates = tmp_path / "price_estimates.parquet"
_write_merged(props)
_write_estimates(estimates, props)
written = join_estimates(props, estimates)
out = pl.read_parquet(props)
assert written == N
assert out.height == N
# Order preserved and the address-half of the key is not left behind.
assert out["order"].to_list() == list(range(N))
assert out["junk"].to_list() == [f"x{i}" for i in range(N)]
assert JOIN_ADDRESS not in out.columns
# Every row carries its own estimate, matched by key despite the shuffle.
assert out["Estimated current price"].to_list() == [float(i * 1000) for i in range(N)]
assert out["Est. price per sqm"].to_list() == [i * 10 for i in range(N)]
assert out["Estimated current price"].null_count() == 0
def test_rerun_is_idempotent(tmp_path: Path):
props = tmp_path / "properties.parquet"
estimates = tmp_path / "price_estimates.parquet"
_write_merged(props)
_write_estimates(estimates, props)
join_estimates(props, estimates)
first = pl.read_parquet(props)
join_estimates(props, estimates) # second run on the augmented file
second = pl.read_parquet(props)
assert second.equals(first)
assert second.columns.count("Estimated current price") == 1
assert second.columns.count("Est. price per sqm") == 1
def test_missing_estimate_is_rejected(tmp_path: Path):
"""A property with no matching estimate (diverged dwelling universe) must
fail loudly rather than silently leave its price null."""
props = tmp_path / "properties.parquet"
estimates = tmp_path / "price_estimates.parquet"
_write_merged(props)
_write_estimates(estimates, props)
# Drop one estimate so a property key is no longer covered.
pl.read_parquet(estimates).head(N - 1).write_parquet(estimates)
with pytest.raises(ValueError, match="no matching estimate"):
join_estimates(props, estimates)
def test_duplicate_key_is_rejected(tmp_path: Path):
props = tmp_path / "properties.parquet"
estimates = tmp_path / "price_estimates.parquet"
_write_merged(props)
_write_estimates(estimates, props)
# Force row 1's key to collide with row 0's.
est = pl.read_parquet(estimates).sort("Estimated current price")
row0 = est.row(0, named=True)
est = est.with_columns(
pl.when(pl.int_range(pl.len()) == 1)
.then(pl.lit(row0["Postcode"]))
.otherwise(pl.col("Postcode"))
.alias("Postcode"),
pl.when(pl.int_range(pl.len()) == 1)
.then(pl.lit(row0[JOIN_ADDRESS]))
.otherwise(pl.col(JOIN_ADDRESS))
.alias(JOIN_ADDRESS),
)
est.write_parquet(estimates)
with pytest.raises(ValueError, match="not unique"):
join_estimates(props, estimates)

View file

@ -10,14 +10,11 @@ from pipeline.transform.merge import (
LISTED_BUILDING_FEATURE, LISTED_BUILDING_FEATURE,
TREE_DENSITY_FEATURE, TREE_DENSITY_FEATURE,
_LISTING_OVERLAY_SOURCES, _LISTING_OVERLAY_SOURCES,
_active_english_postcode_area,
_build_unmatched_listing_seed_rows, _build_unmatched_listing_seed_rows,
_canonical_postcode_expr, _canonical_postcode_expr,
_best_listing_match, _best_listing_match,
_coalesce_direct_epc_columns, _coalesce_direct_epc_columns,
_dedupe_collapsed_properties,
_fill_property_level_no_defaults, _fill_property_level_no_defaults,
_filter_to_active_english_postcodes,
_join_area_side_tables, _join_area_side_tables,
_finalize_listings, _finalize_listings,
_integrate_listings, _integrate_listings,
@ -31,7 +28,6 @@ from pipeline.transform.merge import (
_matched_listed_building_flags, _matched_listed_building_flags,
_postcode_conservation_area_flags, _postcode_conservation_area_flags,
_postcode_listed_building_candidates, _postcode_listed_building_candidates,
_remap_terminated_postcodes,
_split_normal_outputs, _split_normal_outputs,
_tree_density_by_postcode, _tree_density_by_postcode,
_validate_lad_source_coverage, _validate_lad_source_coverage,
@ -39,6 +35,12 @@ from pipeline.transform.merge import (
_validate_postcode_feature_output, _validate_postcode_feature_output,
_validate_property_postcodes, _validate_property_postcodes,
) )
from pipeline.transform.property_base import (
_active_english_postcode_area,
_dedupe_collapsed_properties,
_filter_to_active_english_postcodes,
_remap_terminated_postcodes,
)
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None: def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
@ -115,13 +117,16 @@ def test_tree_density_is_area_level_and_survives_the_split() -> None:
assert TREE_DENSITY_FEATURE not in properties_df.columns assert TREE_DENSITY_FEATURE not in properties_df.columns
def test_crime_columns_are_spatial_counts_not_per_capita() -> None: def test_crime_columns_are_average_annual_counts() -> None:
# Crime is now a raw spatial count per postcode; the per-1k-residents # Crime is the average annual recorded incident count (incidents/yr) over
# variants were dropped along with the LSOA population denominator. # 7-year and 2-year windows; the old per-1,000 "(per 1k/yr, …)" rate columns
assert "Serious crime (avg/yr)" in _AREA_COLUMNS # are gone.
assert "Minor crime (avg/yr)" in _AREA_COLUMNS assert "Serious crime (/yr, 7y)" in _AREA_COLUMNS
assert "Serious crime per 1k residents (avg/yr)" not in _AREA_COLUMNS assert "Serious crime (/yr, 2y)" in _AREA_COLUMNS
assert "Minor crime per 1k residents (avg/yr)" not in _AREA_COLUMNS assert "Minor crime (/yr, 7y)" in _AREA_COLUMNS
assert "Burglary (/yr, 2y)" in _AREA_COLUMNS
assert "Serious crime (avg/yr)" not in _AREA_COLUMNS
assert "Minor crime (avg/yr)" not in _AREA_COLUMNS
def test_active_english_postcode_area_filters_to_active_england() -> None: def test_active_english_postcode_area_filters_to_active_england() -> None:
@ -292,8 +297,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
crime = pl.LazyFrame( crime = pl.LazyFrame(
{ {
"postcode": ["AA1 1AA", "BB2 2BB"], "postcode": ["AA1 1AA", "BB2 2BB"],
"Serious crime (avg/yr)": [1.0, 2.0], "Serious crime (/yr, 7y)": [1.0, 2.0],
"Minor crime (avg/yr)": [3.0, 4.0], "Minor crime (/yr, 7y)": [3.0, 4.0],
} }
) )
joined = _join_area_side_tables( joined = _join_area_side_tables(
@ -343,8 +348,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
crime = pl.LazyFrame( crime = pl.LazyFrame(
{ {
"postcode": ["AB1 2CD", "EF3 4GH"], "postcode": ["AB1 2CD", "EF3 4GH"],
"Serious crime (avg/yr)": [1.0, 2.0], "Serious crime (/yr, 7y)": [1.0, 2.0],
"Minor crime (avg/yr)": [3.0, 4.0], "Minor crime (/yr, 7y)": [3.0, 4.0],
} }
) )
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct # AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
@ -1314,28 +1319,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra}) return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The # Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
# rollup headlines are precomputed values (deliberately NOT the per-type sum, # rollup rate columns are precomputed in crime_spatial and read straight
# which would be 10.0 each) so this test proves the merge consumes the # through unchanged (the merge no longer renames or re-sums them).
# precomputed column rather than re-summing per-type columns.
crime = pl.LazyFrame( crime = pl.LazyFrame(
{ {
"postcode": ["AA1 1AA"], "postcode": ["AA1 1AA"],
"Violence and sexual offences (avg/yr)": [1.0], "Burglary (/yr, 7y)": [3.0],
"Robbery (avg/yr)": [2.0], "Burglary (/yr, 2y)": [3.5],
"Burglary (avg/yr)": [3.0], "Serious crime (/yr, 7y)": [7.5],
"Possession of weapons (avg/yr)": [4.0], "Serious crime (/yr, 2y)": [8.0],
"Anti-social behaviour (avg/yr)": [1.0], "Minor crime (/yr, 7y)": [4.2],
"Criminal damage and arson (avg/yr)": [1.0], "Minor crime (/yr, 2y)": [4.6],
"Shoplifting (avg/yr)": [1.0],
"Bicycle theft (avg/yr)": [1.0],
"Theft from the person (avg/yr)": [1.0],
"Other theft (avg/yr)": [1.0],
"Vehicle crime (avg/yr)": [1.0],
"Public order (avg/yr)": [1.0],
"Drugs (avg/yr)": [1.0],
"Other crime (avg/yr)": [1.0],
"Serious crime (avg/yr)": [7.5],
"Minor crime (avg/yr)": [4.2],
} }
) )
@ -1364,16 +1358,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
by_postcode = { by_postcode = {
row["postcode"]: row row["postcode"]: row
for row in joined.select( for row in joined.select(
"postcode", "serious_crime_avg_yr", "minor_crime_avg_yr" "postcode",
"Serious crime (/yr, 7y)",
"Minor crime (/yr, 2y)",
).iter_rows(named=True) ).iter_rows(named=True)
} }
# Present postcode: rollups are the precomputed headline values, read through # Present postcode: rollup rates pass through unchanged.
# unchanged (NOT the per-type sum of 10.0). assert by_postcode["AA1 1AA"]["Serious crime (/yr, 7y)"] == 7.5
assert by_postcode["AA1 1AA"]["serious_crime_avg_yr"] == 7.5 assert by_postcode["AA1 1AA"]["Minor crime (/yr, 2y)"] == 4.6
assert by_postcode["AA1 1AA"]["minor_crime_avg_yr"] == 4.2 # Missing postcode: rates stay null rather than fabricating 0.0.
# Missing postcode: rollups stay null rather than fabricating 0.0. assert by_postcode["BB2 2BB"]["Serious crime (/yr, 7y)"] is None
assert by_postcode["BB2 2BB"]["serious_crime_avg_yr"] is None assert by_postcode["BB2 2BB"]["Minor crime (/yr, 2y)"] is None
assert by_postcode["BB2 2BB"]["minor_crime_avg_yr"] is None
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None: def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:

View file

@ -1,6 +1,7 @@
mod actual_listings; mod actual_listings;
pub mod area_crime_averages; pub mod area_crime_averages;
pub mod crime_by_year; pub mod crime_by_year;
pub mod crime_records;
mod developments; mod developments;
mod places; mod places;
mod poi; mod poi;
@ -65,6 +66,7 @@ where
pub use actual_listings::{ActualListing, ActualListingData}; pub use actual_listings::{ActualListing, ActualListingData};
pub use area_crime_averages::AreaCrimeAverages; pub use area_crime_averages::AreaCrimeAverages;
pub use crime_by_year::CrimeByYearData; pub use crime_by_year::CrimeByYearData;
pub use crime_records::CrimeRecords;
pub use developments::{DevelopmentData, DevelopmentSite}; pub use developments::{DevelopmentData, DevelopmentSite};
pub use places::{ pub use places::{
compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData, compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,

View file

@ -1,41 +1,61 @@
//! Precomputed per-outcode and per-postcode-sector average crime rates. //! Precomputed per-outcode and per-postcode-sector average crime counts.
//! //!
//! The right pane shows each crime metric's national average (the global //! The right pane shows each crime metric's national average (the global
//! feature-histogram mean). To let users see how an area compares with its //! feature-histogram mean). To let users see how an area compares with its
//! immediate surroundings, we also precompute the mean headline crime rate //! immediate surroundings, we also show the mean average-annual crime count
//! (`"X (avg/yr)"`) across every property in the selection's outcode (e.g. //! (`"X (/yr, 7y|2y)"`) across every property in the selection's outcode
//! `"E14"`) and postcode sector (e.g. `"E14 2"`). //! (e.g. `"E14"`) and postcode sector (e.g. `"E14 2"`).
//! //!
//! Crime figures are constant within a postcode (the pipeline merges them on //! These averages are precomputed by the data pipeline
//! the postcode key), so each postcode's value is read once — from its first //! (`pipeline/transform/area_crime_averages.py`) and loaded here from a side
//! row — and property-weighted by the postcode's row count. That keeps these //! parquet. Crime figures are constant within a postcode, so the pipeline
//! averages on the same property-weighted basis as the national average, so the //! property-weights each postcode's value by its property count — keeping these
//! averages on the same property-weighted basis as the per-selection mean, so the
//! four numbers (this area / sector / outcode / nation) are directly comparable. //! four numbers (this area / sector / outcode / nation) are directly comparable.
use rustc_hash::FxHashMap; use std::path::Path;
/// Crime-feature name suffix that marks an annualised headline-rate column use anyhow::{bail, Context};
/// (e.g. `"Burglary (avg/yr)"`). Stripped to derive the bare type name. use polars::prelude::PlRefPath;
pub const AVG_YR_SUFFIX: &str = " (avg/yr)"; use polars::prelude::*;
use rustc_hash::FxHashMap;
use tracing::info;
use super::run_polars_io;
/// Marker that identifies an average-annual crime-count column (e.g.
/// `"Burglary (/yr, 7y)"`). These are the filterable, area-comparable figures.
/// The full column name is kept as the key, so each per-area mean aligns with the
/// feature the frontend requests.
pub const COUNT_MARKER: &str = " (/yr, ";
/// `scope` column discriminator values written by the pipeline.
const SCOPE_NATIONAL: &str = "national";
const SCOPE_OUTCODE: &str = "outcode";
const SCOPE_SECTOR: &str = "sector";
pub struct AreaCrimeAverages { pub struct AreaCrimeAverages {
/// Bare crime-type names (suffix stripped, e.g. `"Burglary"`), index-aligned /// Full crime feature names (e.g. `"Burglary (/yr, 7y)"`), index-aligned with
/// with the per-area mean vectors. Matches `CrimeYearStats.name`. /// the per-area mean vectors. Matches the feature names the frontend requests,
/// so each NumberLine can look its average up directly.
pub crime_types: Vec<String>, pub crime_types: Vec<String>,
/// National mean headline rate per crime type (index-aligned with /// National mean headline count per crime type (index-aligned with
/// `crime_types`). An EXACT property-weighted mean over every postcode, so it /// `crime_types`). An EXACT property-weighted mean over every postcode, so it
/// shares a basis with `by_outcode`/`by_sector` and the per-selection mean — /// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
/// unlike the histogram-bin national average, which is biased upward for the /// unlike the histogram-bin national average, which is biased upward for the
/// right-skewed crime densities. `NaN` where no postcode has data. /// right-skewed crime counts. `NaN` where no postcode has data.
pub national: Vec<f32>, pub national: Vec<f32>,
/// Outcode (e.g. `"E14"`) → mean headline rate per crime type. `NaN` where /// Outcode (e.g. `"E14"`) → mean headline count per crime type. `NaN` where
/// the outcode has no data for that type. /// the outcode has no data for that type.
pub by_outcode: FxHashMap<String, Vec<f32>>, pub by_outcode: FxHashMap<String, Vec<f32>>,
/// Postcode sector (e.g. `"E14 2"`) → mean headline rate per crime type. /// Postcode sector (e.g. `"E14 2"`) → mean headline count per crime type.
pub by_sector: FxHashMap<String, Vec<f32>>, pub by_sector: FxHashMap<String, Vec<f32>>,
} }
impl AreaCrimeAverages { impl AreaCrimeAverages {
/// Empty table — used only by the test-only `AppState` builder (the real
/// server always loads the precomputed parquet).
#[cfg(test)]
pub fn empty() -> Self { pub fn empty() -> Self {
Self { Self {
crime_types: Vec::new(), crime_types: Vec::new(),
@ -44,4 +64,202 @@ impl AreaCrimeAverages {
by_sector: FxHashMap::default(), by_sector: FxHashMap::default(),
} }
} }
pub fn load(path: &Path) -> anyhow::Result<Self> {
run_polars_io(|| Self::load_inner(path))
}
fn load_inner(path: &Path) -> anyhow::Result<Self> {
info!("Loading area crime averages from {}", path.display());
let pl_path = PlRefPath::try_from_path(path).with_context(|| {
format!(
"Failed to normalize area-crime-averages parquet path {}",
path.display()
)
})?;
let df = LazyFrame::scan_parquet(pl_path, Default::default())
.with_context(|| {
format!(
"Failed to scan area-crime-averages parquet at {}",
path.display()
)
})?
.collect()
.with_context(|| {
format!(
"Failed to read area-crime-averages parquet at {}",
path.display()
)
})?;
// Crime columns are those carrying the count marker; their full names
// are kept (in column order) as the keys, so every per-area mean vector is
// index-aligned with `crime_types` and matches the requested feature name.
let crime_cols: Vec<String> = df
.get_column_names()
.iter()
.filter(|name| name.contains(COUNT_MARKER))
.map(|name| name.to_string())
.collect();
if crime_cols.is_empty() {
bail!(
"area-crime-averages parquet at {} has no '*{COUNT_MARKER}*' count columns",
path.display()
);
}
let crime_types: Vec<String> = crime_cols.clone();
let n = crime_cols.len();
let scope_col = df
.column("scope")
.context("area-crime-averages parquet missing 'scope' column")?
.str()
.context("'scope' column is not a string")?;
let area_col = df
.column("area")
.context("area-crime-averages parquet missing 'area' column")?
.str()
.context("'area' column is not a string")?;
// Hold the casts alive while we borrow `Float32Chunked` views into them.
let casts: Vec<Column> = crime_cols
.iter()
.map(|name| {
df.column(name)
.and_then(|col| col.cast(&DataType::Float32))
.with_context(|| format!("Failed to read crime column '{name}' as f32"))
})
.collect::<anyhow::Result<Vec<_>>>()?;
let crime_views: Vec<&Float32Chunked> = casts
.iter()
.zip(crime_cols.iter())
.map(|(col, name)| {
col.f32()
.with_context(|| format!("crime column '{name}' is not f32 after cast"))
})
.collect::<anyhow::Result<Vec<_>>>()?;
let read_values = |row: usize| -> Vec<f32> {
crime_views
.iter()
.map(|view| view.get(row).unwrap_or(f32::NAN))
.collect()
};
let mut national: Option<Vec<f32>> = None;
let mut by_outcode: FxHashMap<String, Vec<f32>> = FxHashMap::default();
let mut by_sector: FxHashMap<String, Vec<f32>> = FxHashMap::default();
for row in 0..df.height() {
let scope = scope_col
.get(row)
.with_context(|| format!("area-crime-averages row {row} has null scope"))?;
match scope {
SCOPE_NATIONAL => national = Some(read_values(row)),
SCOPE_OUTCODE => {
if let Some(area) = area_col.get(row) {
by_outcode.insert(area.to_string(), read_values(row));
}
}
SCOPE_SECTOR => {
if let Some(area) = area_col.get(row) {
by_sector.insert(area.to_string(), read_values(row));
}
}
other => bail!("area-crime-averages row {row} has unknown scope '{other}'"),
}
}
let national = national.context(
"area-crime-averages parquet has no 'national' row; regenerate it with \
pipeline.transform.area_crime_averages",
)?;
if national.len() != n {
bail!(
"area-crime-averages national row has {} values, expected {n}",
national.len()
);
}
info!(
outcodes = by_outcode.len(),
sectors = by_sector.len(),
crime_types = crime_types.len(),
"Area crime averages loaded"
);
Ok(Self {
crime_types,
national,
by_outcode,
by_sector,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write_fixture(path: &Path) {
// national + one outcode (E14) + one sector (E14 2). Robbery is null for
// the outcode to exercise the NaN round-trip.
let mut df = df!(
"scope" => ["national", "outcode", "sector"],
"area" => ["", "E14", "E14 2"],
"Burglary (/yr, 7y)" => [8.75f32, 12.5, 12.5],
"Robbery (/yr, 7y)" => [Some(1.43f32), None, Some(2.0)],
// A non-crime column must be ignored by the loader.
"Median age" => [40.0f32, 41.0, 42.0],
)
.unwrap();
let mut file = std::fs::File::create(path).unwrap();
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
}
#[test]
fn load_round_trips_national_outcode_sector() {
let dir = std::env::temp_dir().join(format!("acavg-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("area_crime_averages.parquet");
write_fixture(&path);
let avgs = AreaCrimeAverages::load(&path).unwrap();
// Crime columns are discovered by the count marker; "Median age" is not.
assert_eq!(
avgs.crime_types,
vec!["Burglary (/yr, 7y)", "Robbery (/yr, 7y)"]
);
assert_eq!(avgs.national, vec![8.75, 1.43]);
let e14 = avgs.by_outcode.get("E14").unwrap();
assert_eq!(e14[0], 12.5);
// The null robbery value becomes NaN, which the consumer drops to None.
assert!(e14[1].is_nan());
let e14_2 = avgs.by_sector.get("E14 2").unwrap();
assert_eq!(e14_2, &vec![12.5, 2.0]);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn load_rejects_parquet_without_national_row() {
let dir = std::env::temp_dir().join(format!("acavg-nonat-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("no_national.parquet");
let mut df = df!(
"scope" => ["outcode"],
"area" => ["E14"],
"Burglary (/yr, 7y)" => [12.5f32],
)
.unwrap();
let mut file = std::fs::File::create(&path).unwrap();
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
assert!(AreaCrimeAverages::load(&path).is_err());
std::fs::remove_dir_all(&dir).ok();
}
} }

View file

@ -0,0 +1,534 @@
//! Individual police.uk crime records (last 7 years) backing the right pane's
//! "individual crimes" list and the `/api/crime-records` endpoint.
//!
//! This table is enormous — ~500M rows, because each incident is replicated to
//! every postcode whose buffer covers it (see [`gather`](CrimeRecords::gather)),
//! so it is NOT held as a `Vec<struct>`: each field is a flat columnar
//! [`SpillVec`] (mmap-backed and kernel-reclaimable when `--spill-dir` is set),
//! small string fields are dictionary-encoded, and the parquet is pre-sorted by
//! postcode so each postcode's records are a contiguous `[start, start+count)`
//! slice located via a CSR-style offset index. Resident RSS is ~0 until records
//! are actually read.
//!
//! At ~500M rows the parquet's string columns (postcode/type/location/outcome)
//! decode to tens of GB if read whole, so the loader never materialises the
//! whole `DataFrame`: it streams the file in bounded row-count chunks (only the
//! row groups overlapping each slice are decoded) and writes each column
//! straight into its (optionally spilled) backing store via [`SpillVecBuilder`],
//! keeping the transient footprint to one chunk plus the index maps.
use std::fs::File;
use std::path::Path;
use anyhow::{bail, Context};
use lasso::{Rodeo, RodeoReader, Spur};
use polars::prelude::*;
use rustc_hash::FxHashMap;
use tracing::info;
use super::run_polars_io;
use super::spill::{SpillVec, SpillVecBuilder};
/// Rows decoded per streaming slice. `with_slice` decodes only the row groups
/// overlapping the slice, so the transient decode is roughly one chunk's worth
/// (~tens of MB at the writer's ~123k-row groups) instead of the tens-of-GB
/// whole-file `DataFrame`. The bound's only dependency on file layout is a
/// reasonable input row-group size, which our pipeline writer produces.
const CHUNK_ROWS: usize = 2_000_000;
/// A resolved view of one record (strings dereferenced from the dictionaries).
pub struct CrimeRecordView<'a> {
/// `year * 12 + (month - 1)`.
pub month_index: u32,
pub crime_type: &'a str,
pub outcome: Option<&'a str>,
pub location: Option<&'a str>,
pub lat: f32,
pub lon: f32,
}
pub struct CrimeRecords {
month: SpillVec<u32>,
ctype: SpillVec<u8>,
outcome: SpillVec<u8>,
location: SpillVec<Spur>,
lat: SpillVec<f32>,
lon: SpillVec<f32>,
/// Dictionary for `ctype` (bare crime type names, e.g. "Burglary").
crime_type_dict: Vec<String>,
/// Dictionary for `outcome`; index 0 is the empty/unknown sentinel.
outcome_dict: Vec<String>,
/// Resolver for the interned `location` strings (`""` means withheld).
location_resolver: RodeoReader,
/// Postcode → `(start, count)` into the columnar arrays (records for a
/// postcode are contiguous because the parquet is sorted by postcode).
by_postcode: FxHashMap<String, (u32, u32)>,
}
impl CrimeRecords {
#[cfg(test)]
pub fn empty() -> Self {
Self {
month: SpillVec::owned(Vec::new()),
ctype: SpillVec::owned(Vec::new()),
outcome: SpillVec::owned(Vec::new()),
location: SpillVec::owned(Vec::new()),
lat: SpillVec::owned(Vec::new()),
lon: SpillVec::owned(Vec::new()),
crime_type_dict: Vec::new(),
outcome_dict: vec![String::new()],
location_resolver: Rodeo::default().into_reader(),
by_postcode: FxHashMap::default(),
}
}
/// Number of records stored for a postcode (0 if none).
pub fn total_for(&self, postcode: &str) -> u32 {
self.by_postcode.get(postcode).map_or(0, |&(_, c)| c)
}
/// Resolve a record index to a borrowing view.
pub fn view(&self, idx: u32) -> CrimeRecordView<'_> {
let i = idx as usize;
let outcome_idx = self.outcome[i] as usize;
let outcome = self
.outcome_dict
.get(outcome_idx)
.filter(|s| !s.is_empty())
.map(String::as_str);
let location = self.location_resolver.resolve(&self.location[i]);
CrimeRecordView {
month_index: self.month[i],
crime_type: self
.crime_type_dict
.get(self.ctype[i] as usize)
.map_or("", String::as_str),
outcome,
location: (!location.is_empty()).then_some(location),
lat: self.lat[i],
lon: self.lon[i],
}
}
/// Record indices across `postcodes`, newest first, optionally restricted to
/// months `>= since_month`. These are exactly the incidents counted for the
/// selected postcodes — for a single postcode that is its precise incident
/// list; for a multi-postcode selection a boundary incident counted for
/// several postcodes appears once per postcode, matching the count. We do not
/// de-duplicate because police.uk snaps many genuinely distinct incidents
/// (especially anti-social behaviour) to the same point/month and provides no
/// per-incident id to tell a true duplicate from two real incidents apart.
pub fn gather(&self, postcodes: &[&str], since_month: Option<u32>) -> Vec<u32> {
let month = self.month.as_slice();
let mut out: Vec<u32> = Vec::new();
for pc in postcodes {
let Some(&(start, count)) = self.by_postcode.get(*pc) else {
continue;
};
for i in start..start + count {
if since_month.map_or(true, |s| month[i as usize] >= s) {
out.push(i);
}
}
}
out.sort_unstable_by(|&a, &b| month[b as usize].cmp(&month[a as usize]));
out
}
pub fn load(path: &Path, spill_dir: Option<&Path>) -> anyhow::Result<Self> {
run_polars_io(|| Self::load_inner(path, spill_dir, CHUNK_ROWS))
}
fn load_inner(
path: &Path,
spill_dir: Option<&Path>,
chunk_rows: usize,
) -> anyhow::Result<Self> {
// A zero chunk size would loop forever; the public entry point passes the
// const, but tests parameterise this to exercise chunk boundaries.
let chunk_rows = chunk_rows.max(1);
info!("Loading crime records from {}", path.display());
// Read the footer once for the row count, and keep it to hand to every
// per-chunk reader so the 2.9MB metadata is never re-parsed.
let metadata = {
let file = File::open(path).with_context(|| {
format!("Failed to open crime-records parquet at {}", path.display())
})?;
ParquetReader::new(file)
.get_metadata()
.with_context(|| {
format!("Failed to read crime-records parquet metadata at {}", path.display())
})?
.clone()
};
let n = metadata.num_rows;
// Record indices are stored as `u32` (and `by_postcode` holds `(start,
// count)` as `u32`), so the table must fit in that index space.
if n > u32::MAX as usize {
bail!("crime-records parquet has {n} rows, exceeding the u32 record-index limit");
}
// Columns that, when spilling, are written straight into mmap-backed files
// as we stream — so the ~9GB of columnar data never lands on the heap.
let mut month = SpillVecBuilder::<u32>::with_len(n, spill_dir, "crime_month")?;
let mut ctype = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_ctype")?;
let mut outcome = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_outcome")?;
let mut location = SpillVecBuilder::<Spur>::with_len(n, spill_dir, "crime_location")?;
let mut lat = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lat")?;
let mut lon = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lon")?;
let mut crime_type_dict: Vec<String> = Vec::new();
let mut type_index: FxHashMap<String, u8> = FxHashMap::default();
// Outcome index 0 is the empty/unknown sentinel.
let mut outcome_dict: Vec<String> = vec![String::new()];
let mut outcome_index: FxHashMap<String, u8> = FxHashMap::default();
let mut rodeo = Rodeo::default();
let empty_spur = rodeo.get_or_intern("");
let mut by_postcode: FxHashMap<String, (u32, u32)> = FxHashMap::default();
let mut cur_pc: Option<String> = None;
let mut cur_start: u32 = 0;
// Absolute record index across all chunks; drives both the CSR index and
// the column builders' write order.
let mut global_row: u32 = 0;
let columns: Vec<String> = ["postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"]
.iter()
.map(|s| s.to_string())
.collect();
let mut offset = 0usize;
while offset < n {
let len = chunk_rows.min(n - offset);
let file = File::open(path).with_context(|| {
format!("Failed to open crime-records parquet at {}", path.display())
})?;
let mut reader = ParquetReader::new(file);
reader.set_metadata(metadata.clone());
// `with_slice` only decodes the row groups overlapping `[offset,
// offset+len)` (the file is memory-mapped, so untouched groups are
// never faulted in), capping the transient decode to one chunk.
let df = reader
.with_columns(Some(columns.clone()))
.with_slice(Some((offset, len)))
.finish()
.with_context(|| {
format!(
"Failed to read crime-records rows [{offset}, {}) from {}",
offset + len,
path.display()
)
})?;
let postcode_col = df
.column("postcode")
.context("crime-records parquet missing 'postcode'")?
.str()
.context("'postcode' is not a string")?;
let month_col = df
.column("month_index")
.context("crime-records parquet missing 'month_index'")?
.cast(&DataType::Int32)
.context("'month_index' not castable to i32")?;
let month_ca = month_col.i32().context("'month_index' is not i32")?;
// Months are `year*12 + month0` (~24_000), always positive. A null or
// non-positive value means a corrupt parquet; fail loudly rather than
// silently clamping it to 0 and later rendering it as "0000-01".
if month_ca.null_count() > 0 {
bail!("crime-records 'month_index' has null values (corrupt parquet)");
}
match month_ca.min() {
Some(m) if m > 0 => {}
_ => bail!("crime-records 'month_index' must be a positive year*12+month index"),
}
let type_col = df
.column("crime_type")
.context("crime-records parquet missing 'crime_type'")?
.str()
.context("'crime_type' is not a string")?;
let location_col = df
.column("location")
.context("crime-records parquet missing 'location'")?
.str()
.context("'location' is not a string")?;
let outcome_col = df
.column("outcome")
.context("crime-records parquet missing 'outcome'")?
.str()
.context("'outcome' is not a string")?;
let lat_col = df
.column("lat")
.context("crime-records parquet missing 'lat'")?
.cast(&DataType::Float32)?;
let lat_ca = lat_col.f32().context("'lat' is not f32")?;
let lon_col = df
.column("lon")
.context("crime-records parquet missing 'lon'")?
.cast(&DataType::Float32)?;
let lon_ca = lon_col.f32().context("'lon' is not f32")?;
let height = df.height();
for row in 0..height {
// CSR index: the parquet is sorted by postcode, so a change in the
// postcode value (across chunk boundaries too) closes the previous
// run and opens a new one.
let pc = postcode_col
.get(row)
.with_context(|| {
format!("crime-records row {} has null postcode", offset + row)
})?
.trim();
if cur_pc.as_deref() != Some(pc) {
if let Some(prev) = cur_pc.take() {
by_postcode.insert(prev, (cur_start, global_row - cur_start));
}
cur_pc = Some(pc.to_string());
cur_start = global_row;
}
month.push(month_ca.get(row).unwrap() as u32);
let ty = type_col.get(row).unwrap_or("");
let ty_id = match type_index.get(ty) {
Some(&id) => id,
None => {
let id = u8::try_from(crime_type_dict.len())
.context("more than 256 distinct crime types")?;
crime_type_dict.push(ty.to_string());
type_index.insert(ty.to_string(), id);
id
}
};
ctype.push(ty_id);
let oc = outcome_col.get(row).unwrap_or("");
let oc_id = if oc.is_empty() {
0
} else {
match outcome_index.get(oc) {
Some(&id) => id,
None => {
let id = u8::try_from(outcome_dict.len())
.context("more than 256 distinct outcomes")?;
outcome_dict.push(oc.to_string());
outcome_index.insert(oc.to_string(), id);
id
}
}
};
outcome.push(oc_id);
let loc = location_col.get(row).unwrap_or("");
location.push(if loc.is_empty() {
empty_spur
} else {
rodeo.get_or_intern(loc)
});
lat.push(lat_ca.get(row).unwrap_or(f32::NAN));
lon.push(lon_ca.get(row).unwrap_or(f32::NAN));
global_row += 1;
}
offset += len;
}
if let Some(prev) = cur_pc.take() {
by_postcode.insert(prev, (cur_start, global_row - cur_start));
}
debug_assert_eq!(global_row as usize, n, "streamed fewer rows than the parquet declares");
let records = Self {
month: month.finish()?,
ctype: ctype.finish()?,
outcome: outcome.finish()?,
location: location.finish()?,
lat: lat.finish()?,
lon: lon.finish()?,
crime_type_dict,
outcome_dict,
location_resolver: rodeo.into_reader(),
by_postcode,
};
info!(
records = n,
postcodes = records.by_postcode.len(),
crime_types = records.crime_type_dict.len(),
outcomes = records.outcome_dict.len(),
"Crime records loaded"
);
Ok(records)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write_fixture(path: &Path) {
// Two postcodes, postcode-sorted. AA1 1AA has 3 records across two
// months (a null outcome and a null location), BB2 2BB has 1.
let mut df = df!(
"postcode" => ["AA1 1AA", "AA1 1AA", "AA1 1AA", "BB2 2BB"],
"month_index" => [24300i32, 24300, 24290, 24305],
"crime_type" => ["Burglary", "Burglary", "Robbery", "Drugs"],
"location" => [Some("On or near A St"), Some("On or near A St"), None, Some("On or near B Rd")],
"outcome" => [Some("Under investigation"), None, Some("Court result"), None],
"lat" => [51.5f32, 51.5, 51.6, 52.0],
"lon" => [-0.1f32, -0.1, -0.2, -1.0],
)
.unwrap();
let mut file = std::fs::File::create(path).unwrap();
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
}
#[test]
fn loads_indexes_and_gathers() {
let dir = std::env::temp_dir().join(format!("crimerec-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("records.parquet");
write_fixture(&path);
let recs = CrimeRecords::load(&path, None).unwrap();
assert_eq!(recs.total_for("AA1 1AA"), 3);
assert_eq!(recs.total_for("BB2 2BB"), 1);
assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
// Newest-first across the two postcodes.
let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
assert_eq!(all.len(), 4);
let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
assert_eq!(months, vec![24305, 24300, 24300, 24290]);
// `since` window filter (keep months >= 24300).
assert_eq!(recs.gather(&["AA1 1AA"], Some(24300)).len(), 2);
// String resolution + null handling.
let robbery = all
.iter()
.map(|&i| recs.view(i))
.find(|v| v.crime_type == "Robbery")
.unwrap();
assert_eq!(robbery.outcome, Some("Court result"));
assert_eq!(robbery.location, None); // null location → None
// Two records have a null outcome (an AA1 Burglary and the BB2 Drugs).
let null_outcomes = all
.iter()
.map(|&i| recs.view(i))
.filter(|v| v.outcome.is_none())
.count();
assert_eq!(null_outcomes, 2);
std::fs::remove_dir_all(&dir).ok();
}
/// The CSR per-postcode index and the column builders must compose correctly
/// across streaming chunk boundaries — including a postcode run split between
/// two chunks. Forces `chunk_rows = 2` over the 4-row fixture so AA1 1AA's
/// three records straddle the boundary (rows 0,1 in chunk 0; row 2 in chunk 1)
/// and is exercised both heap-backed (no spill) and mmap-backed (spill).
#[test]
fn streams_across_chunk_boundaries() {
let base = std::env::temp_dir().join(format!("crimerec-chunk-{}", std::process::id()));
std::fs::create_dir_all(&base).unwrap();
let path = base.join("records.parquet");
write_fixture(&path);
let spill = base.join("spill");
std::fs::create_dir_all(&spill).unwrap();
for spill_dir in [None, Some(spill.as_path())] {
let recs = CrimeRecords::load_inner(&path, spill_dir, 2).unwrap();
// Counts match regardless of how the runs were split across chunks.
assert_eq!(recs.total_for("AA1 1AA"), 3);
assert_eq!(recs.total_for("BB2 2BB"), 1);
assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
// Full gather, newest-first, identical to the single-chunk load.
let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
assert_eq!(all.len(), 4);
let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
assert_eq!(months, vec![24305, 24300, 24300, 24290]);
// The run that straddled the boundary still resolves its strings.
let robbery = all
.iter()
.map(|&i| recs.view(i))
.find(|v| v.crime_type == "Robbery")
.unwrap();
assert_eq!(robbery.outcome, Some("Court result"));
assert_eq!(robbery.location, None);
}
std::fs::remove_dir_all(&base).ok();
}
/// Peak/resident RSS in MiB from `/proc/self/status` (Linux only).
fn rss_mib() -> (f64, f64) {
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
let field = |key: &str| -> f64 {
status
.lines()
.find(|l| l.starts_with(key))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|kb| kb.parse::<f64>().ok())
.map_or(0.0, |kb| kb / 1024.0)
};
(field("VmHWM:"), field("VmRSS:"))
}
/// Manual, real-data smoke test: load the actual ~500M-row parquet and report
/// peak RSS, proving the streaming + spill load completes without the
/// tens-of-GB `DataFrame` materialisation that OOMed the old `.collect()`.
///
/// Run with:
/// PPC_REAL_CRIME_RECORDS=/path/to/crime_records.parquet \
/// cargo test --bins -- --ignored --nocapture real_crime_records_load_is_bounded
#[test]
#[ignore = "needs the full crime_records.parquet; run manually"]
fn real_crime_records_load_is_bounded() {
let path = std::env::var("PPC_REAL_CRIME_RECORDS")
.unwrap_or_else(|_| "../property-data/crime_records.parquet".to_string());
let path = Path::new(&path);
if !path.exists() {
eprintln!("skipping: {} not found", path.display());
return;
}
let spill = std::env::var("PPC_REAL_SPILL")
.unwrap_or_else(|_| "../.tmp/crime-spill-realtest".to_string());
let spill = Path::new(&spill);
std::fs::create_dir_all(spill).unwrap();
let (hwm_before, _rss_before) = rss_mib();
let start = std::time::Instant::now();
let recs = CrimeRecords::load(path, Some(spill)).unwrap();
let elapsed = start.elapsed();
let (hwm_after, rss_after) = rss_mib();
let total: u64 = recs.by_postcode.values().map(|&(_, c)| c as u64).sum();
eprintln!(
"loaded {} records across {} postcodes in {:.1}s | RSS peak {:.0}->{:.0} MiB (Δ{:.0}) resident now {:.0} MiB",
total,
recs.by_postcode.len(),
elapsed.as_secs_f64(),
hwm_before,
hwm_after,
hwm_after - hwm_before,
rss_after,
);
assert!(recs.by_postcode.len() > 0, "expected at least one postcode");
assert!(total > 0, "expected at least one record");
// The old `.collect()` decoded all rows' string columns at once (tens of
// GB). Streaming must keep the peak growth far below that; a generous 20GiB
// ceiling still proves we never materialise the whole file.
assert!(
hwm_after - hwm_before < 20_480.0,
"peak RSS grew by {:.0} MiB during load — streaming/spill not bounding memory",
hwm_after - hwm_before
);
std::fs::remove_dir_all(spill).ok();
}
}

View file

@ -26,9 +26,7 @@ use rustc_hash::FxHashMap;
use serde::Serialize; use serde::Serialize;
use crate::consts::NAN_U16; use crate::consts::NAN_U16;
use crate::data::area_crime_averages::{AreaCrimeAverages, AVG_YR_SUFFIX};
use crate::data::spill::SpillVec; use crate::data::spill::SpillVec;
use crate::utils::{postcode_outcode, postcode_sector};
#[derive(Serialize, Clone)] #[derive(Serialize, Clone)]
pub struct RenovationEvent { pub struct RenovationEvent {
@ -226,109 +224,6 @@ impl PropertyData {
num_numeric: self.num_numeric, num_numeric: self.num_numeric,
} }
} }
/// Precompute mean headline crime rates nationally and per outcode / postcode
/// sector.
///
/// Crime values are identical for every property in a postcode (the pipeline
/// merges them on the postcode key), so each postcode is sampled once from
/// its first row and property-weighted by its row count. All three scopes use
/// the same exact property-weighted estimator over the same row universe as
/// the per-selection mean, so the four numbers shown in a crime row (this
/// selection / sector / outcode / nation) are directly comparable — without
/// the upward bias of the histogram-bin national average.
pub fn compute_area_crime_averages(&self) -> AreaCrimeAverages {
// Crime headline columns are exactly the " (avg/yr)" features.
let crime_indices: Vec<usize> = self
.feature_names
.iter()
.enumerate()
.filter(|(_, name)| name.ends_with(AVG_YR_SUFFIX))
.map(|(idx, _)| idx)
.collect();
if crime_indices.is_empty() {
return AreaCrimeAverages::empty();
}
let crime_types: Vec<String> = crime_indices
.iter()
.map(|&idx| {
self.feature_names[idx]
.strip_suffix(AVG_YR_SUFFIX)
.unwrap_or(&self.feature_names[idx])
.to_string()
})
.collect();
let n = crime_indices.len();
// (weighted value sum, weight) accumulators per crime type.
let mut nat_sums = vec![0.0f64; n];
let mut nat_weights = vec![0u64; n];
let mut out_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
let mut sec_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
for (key, rows) in &self.postcode_row_index {
let Some(&first) = rows.first() else { continue };
let count = rows.len() as u64;
let postcode = self.postcode_interner.resolve(key);
let outcode = postcode_outcode(postcode);
let sector = postcode_sector(postcode);
for (j, &fi) in crime_indices.iter().enumerate() {
// A NaN value is "no crime data for this postcode" — skip it so
// it dilutes neither the sum nor the weight (a genuine gap, not
// a zero), exactly as the global histogram excludes it.
let value = self.get_feature(first as usize, fi);
if !value.is_finite() {
continue;
}
let weighted = value as f64 * count as f64;
// National counts every postcode (the population the global mean
// is built over); outcode/sector only when the postcode parses.
nat_sums[j] += weighted;
nat_weights[j] += count;
if let Some(outcode) = outcode {
let acc = out_acc
.entry(outcode.to_string())
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
acc.0[j] += weighted;
acc.1[j] += count;
}
if let Some(sector) = sector {
let acc = sec_acc
.entry(sector.to_string())
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
acc.0[j] += weighted;
acc.1[j] += count;
}
}
}
let means_of = |sums: &[f64], weights: &[u64]| -> Vec<f32> {
sums.iter()
.zip(weights.iter())
.map(|(&sum, &weight)| {
if weight == 0 {
f32::NAN
} else {
(sum / weight as f64) as f32
}
})
.collect()
};
let finalize =
|acc: FxHashMap<String, (Vec<f64>, Vec<u64>)>| -> FxHashMap<String, Vec<f32>> {
acc.into_iter()
.map(|(area, (sums, weights))| (area, means_of(&sums, &weights)))
.collect()
};
AreaCrimeAverages {
crime_types,
national: means_of(&nat_sums, &nat_weights),
by_outcode: finalize(out_acc),
by_sector: finalize(sec_acc),
}
}
} }
#[cfg(test)] #[cfg(test)]

View file

@ -133,6 +133,138 @@ impl SpillVec<u16> {
} }
} }
/// Builds a [`SpillVec<T>`] incrementally by `push`ing elements one at a time,
/// when the final length is known up front but the values arrive in a stream.
///
/// When a spill `dir` is set (and the length is non-zero) the backing store is a
/// pre-sized, memory-mapped file and each pushed element is written straight into
/// it — so the array never exists as a heap `Vec` and is never copied a second
/// time on finalisation, unlike [`SpillVec::maybe_spill`], which takes an
/// already-built `Vec` and so needs the whole thing resident first. Without a
/// spill dir it accumulates into an owned `Vec` (production behaviour, identical
/// resident cost to the old `Vec::with_capacity` + `maybe_spill`). This is what
/// lets the ~500M-row crime-records columns load without a multi-GB heap spike.
///
/// Exactly `len` elements must be pushed: pushing more panics, and finishing with
/// fewer is an error.
pub struct SpillVecBuilder<T: SpillElem> {
backing: Builder<T>,
}
enum Builder<T: SpillElem> {
Owned { values: Vec<T>, len: usize },
Mapped {
map: MmapMut,
len: usize,
cursor: usize,
label: &'static str,
_marker: PhantomData<T>,
},
}
impl<T: SpillElem> SpillVecBuilder<T> {
/// Create a builder for exactly `len` elements. Spills to `dir` when it is set
/// and `len > 0` (a zero-length mmap is invalid); otherwise reserves an owned
/// `Vec`. `label` names the backing file for diagnostics.
pub fn with_len(len: usize, dir: Option<&Path>, label: &'static str) -> anyhow::Result<Self> {
match dir {
Some(dir) if len > 0 => {
let byte_len = len * std::mem::size_of::<T>();
let file = anon_file(dir, label)?;
allocate_spill_file(&file, byte_len, label)?;
// SAFETY: `file` is a freshly-created, exclusively-owned regular
// file sized to exactly `byte_len`; no other mapping aliases it.
let map = unsafe { MmapMut::map_mut(&file) }.with_context(|| {
format!("mapping spill file for '{label}' ({byte_len} bytes)")
})?;
Ok(Self {
backing: Builder::Mapped {
map,
len,
cursor: 0,
label,
_marker: PhantomData,
},
})
}
_ => Ok(Self {
backing: Builder::Owned {
values: Vec::with_capacity(len),
len,
},
}),
}
}
/// Append one element. Panics if more than the declared `len` elements are
/// pushed — enforced identically on both backings so a streaming bug fails
/// the same way in production (owned) as in dev (spilled).
#[inline]
pub fn push(&mut self, value: T) {
match &mut self.backing {
Builder::Owned { values, len } => {
assert!(
values.len() < *len,
"SpillVecBuilder overflow: pushed more than {len} elements"
);
values.push(value);
}
Builder::Mapped {
map, len, cursor, ..
} => {
assert!(
*cursor < *len,
"SpillVecBuilder overflow: pushed more than {len} elements"
);
// SAFETY: an mmap base is page-aligned (hence aligned for `T`); the
// mapping holds exactly `len * size_of::<T>()` bytes and `cursor <
// len`, so this writes one in-bounds, aligned `T`. `T: SpillElem`
// is `Copy` and padding-free, so the stored byte image is fully
// defined and reads back as the same value.
unsafe { map.as_mut_ptr().cast::<T>().add(*cursor).write(value) };
*cursor += 1;
}
}
}
/// Seal the builder into a read-only [`SpillVec`]. Exactly the declared `len`
/// elements must have been pushed, on both backings.
pub fn finish(self) -> anyhow::Result<SpillVec<T>> {
match self.backing {
Builder::Owned { values, len } => {
if values.len() != len {
anyhow::bail!(
"spill builder finished with {} of {len} elements written",
values.len()
);
}
Ok(SpillVec::Owned(values))
}
Builder::Mapped {
map,
len,
cursor,
label,
..
} => {
if cursor != len {
anyhow::bail!(
"spill builder for '{label}' finished with {cursor} of {len} elements written"
);
}
let map = map
.make_read_only()
.with_context(|| format!("sealing spill file for '{label}'"))?;
Ok(SpillVec::Mapped {
map,
len,
_marker: PhantomData,
})
}
}
}
}
impl<T: SpillElem> std::ops::Deref for SpillVec<T> { impl<T: SpillElem> std::ops::Deref for SpillVec<T> {
type Target = [T]; type Target = [T];
@ -327,4 +459,105 @@ mod tests {
assert!(matches!(mapped, SpillVec::Owned(_))); assert!(matches!(mapped, SpillVec::Owned(_)));
assert!(mapped.is_empty()); assert!(mapped.is_empty());
} }
#[test]
fn builder_streams_identically_owned_and_mapped() {
let values: Vec<u32> = (0..50_000u32).map(|n| n.wrapping_mul(2_246_822_519)).collect();
// Owned path (no spill dir): pushes accumulate into a heap Vec.
let mut owned = SpillVecBuilder::<u32>::with_len(values.len(), None, "u32_owned").unwrap();
for &v in &values {
owned.push(v);
}
let owned = owned.finish().unwrap();
assert!(matches!(owned, SpillVec::Owned(_)));
assert_eq!(&*owned, values.as_slice());
// Mapped path (spill dir): pushes write straight into the mmap, no heap copy.
let dir = TempDir::new("builder-u32");
let mut mapped =
SpillVecBuilder::<u32>::with_len(values.len(), Some(dir.path()), "u32_mapped").unwrap();
for &v in &values {
mapped.push(v);
}
let mapped = mapped.finish().unwrap();
assert!(matches!(mapped, SpillVec::Mapped { .. }));
// The mmap-backed slice must be byte-identical to the streamed input.
assert_eq!(&*mapped, values.as_slice());
}
#[test]
fn builder_spurs_survive_the_mmap_roundtrip() {
// Spur is a NonZeroU32 niche type — exercises the streamed write path for
// the crime-records `location` column.
let mut rodeo = lasso::Rodeo::default();
let keys: Vec<lasso::Spur> = (0..3000)
.map(|n| rodeo.get_or_intern(format!("loc-{n}")))
.collect();
let dir = TempDir::new("builder-spur");
let mut builder =
SpillVecBuilder::<lasso::Spur>::with_len(keys.len(), Some(dir.path()), "spurs").unwrap();
for &k in &keys {
builder.push(k);
}
let mapped = builder.finish().unwrap();
assert!(matches!(mapped, SpillVec::Mapped { .. }));
assert_eq!(&*mapped, keys.as_slice());
let reader = rodeo.into_resolver();
for (idx, &key) in mapped.iter().enumerate() {
assert_eq!(reader.resolve(&key), format!("loc-{idx}"));
}
}
#[test]
fn builder_zero_len_stays_owned() {
let dir = TempDir::new("builder-empty");
let builder = SpillVecBuilder::<u32>::with_len(0, Some(dir.path()), "empty").unwrap();
let v = builder.finish().unwrap();
// A zero-length mmap is invalid, so empties fall back to an owned Vec.
assert!(matches!(v, SpillVec::Owned(_)));
assert!(v.is_empty());
}
#[test]
fn builder_underfill_is_an_error() {
let dir = TempDir::new("builder-underfill");
let mut builder =
SpillVecBuilder::<u32>::with_len(4, Some(dir.path()), "underfill").unwrap();
builder.push(1);
builder.push(2);
// Sealing a spilling builder before all declared elements are written fails
// rather than exposing uninitialised mmap tail bytes as valid data.
assert!(builder.finish().is_err());
}
#[test]
#[should_panic(expected = "overflow")]
fn builder_overfill_panics() {
let dir = TempDir::new("builder-overfill");
let mut builder = SpillVecBuilder::<u32>::with_len(2, Some(dir.path()), "overfill").unwrap();
builder.push(1);
builder.push(2);
builder.push(3); // one past the declared length
}
#[test]
fn builder_owned_underfill_is_an_error() {
// The owned (no-spill, production) path enforces the declared length too,
// so a streaming bug can't silently yield a short array in release builds.
let mut builder = SpillVecBuilder::<u32>::with_len(4, None, "owned-underfill").unwrap();
builder.push(1);
builder.push(2);
assert!(builder.finish().is_err());
}
#[test]
#[should_panic(expected = "overflow")]
fn builder_owned_overfill_panics() {
let mut builder = SpillVecBuilder::<u32>::with_len(2, None, "owned-overfill").unwrap();
builder.push(1);
builder.push(2);
builder.push(3); // one past the declared length
}
} }

View file

@ -68,6 +68,59 @@ pub struct FeatureGroup {
pub features: &'static [Feature], pub features: &'static [Feature],
} }
/// Expand each crime type into its two filterable features: a 7-year and a
/// 2-year window. Each is the average number of recorded incidents per year (the
/// raw, absolute count — no per-area or per-capita normalisation). The names must
/// match the `"{type} (/yr, 7y|2y)"` columns written by `crime_spatial`. The
/// per-incident records are NOT a feature (they are a display-only side table the
/// server loads directly), so they never appear here and are not filterable.
macro_rules! crime_features {
($( ($base:literal, $blurb:literal) ),+ $(,)?) => {
&[ $(
Feature::Numeric(FeatureConfig {
name: concat!($base, " (/yr, 7y)"),
bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
step: 0.1,
description: concat!($blurb, " — average recorded incidents per year (last 7 years)"),
detail: concat!(
$blurb,
", as the average number of recorded incidents per year, over the last \
7 years. Counted from police.uk street-level crime points (anonymised, \
snapped to nearby map points) that fall near the postcode boundary \
the raw, absolute count, with no per-area or per-capita adjustment. \
Computed over the months the local police force actually published; \
known force gaps (e.g. Greater Manchester since mid-2019) are excluded, \
not counted as zero crime."
),
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: concat!($base, " (/yr, 2y)"),
bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
step: 0.1,
description: concat!($blurb, " — average recorded incidents per year (last 2 years)"),
detail: concat!(
$blurb,
", as the average number of recorded incidents per year, over the last \
2 years a more recent but noisier window than the 7-year figure. From \
police.uk street-level crime points near the postcode boundary (the raw, \
absolute count), over the months the local force published (gaps \
excluded, not zeroed)."
),
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
)+ ]
};
}
pub static FEATURE_GROUPS: &[FeatureGroup] = &[ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
FeatureGroup { FeatureGroup {
name: "Properties", name: "Properties",
@ -472,247 +525,41 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
}, },
FeatureGroup { FeatureGroup {
name: "Crime", name: "Crime",
features: &[ features: crime_features![
Feature::Numeric(FeatureConfig { (
name: "Serious crime (avg/yr)", "Serious crime",
bounds: Bounds::Percentile { "Serious crime — violence, robbery, burglary and weapons possession — near the postcode"
low: 2.0, ),
high: 98.0, (
}, "Minor crime",
step: 1.0, "Lower-severity crime — anti-social behaviour, theft, criminal damage, drugs and public order — near the postcode"
description: "Relative density of serious crime categories near the postcode", ),
detail: "Combined density of violence, robbery, burglary, and weapons possession near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.", (
source: "crime", "Violence and sexual offences",
prefix: "", "Violence and sexual offences (assault, harassment, sexual offences) near the postcode"
suffix: "", ),
raw: false, ("Burglary", "Burglary (residential and commercial) near the postcode"),
absolute: false, ("Robbery", "Robbery (theft with force or threat) near the postcode"),
}), ("Vehicle crime", "Vehicle crime (theft of and from vehicles) near the postcode"),
Feature::Numeric(FeatureConfig { ("Anti-social behaviour", "Anti-social behaviour near the postcode"),
name: "Minor crime (avg/yr)", ("Criminal damage and arson", "Criminal damage and arson near the postcode"),
bounds: Bounds::Percentile { (
low: 2.0, "Other theft",
high: 98.0, "Other theft (not burglary, vehicle, shoplifting or bicycle theft) near the postcode"
}, ),
step: 1.0, (
description: "Relative density of minor crime categories near the postcode", "Theft from the person",
detail: "Combined density of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.", "Theft from the person (pickpocketing, bag snatching) near the postcode"
source: "crime", ),
prefix: "", ("Shoplifting", "Shoplifting near the postcode"),
suffix: "", ("Bicycle theft", "Bicycle theft near the postcode"),
raw: false, ("Drugs", "Drug offences (possession and trafficking) near the postcode"),
absolute: false, ("Possession of weapons", "Possession of weapons near the postcode"),
}), (
Feature::Numeric(FeatureConfig { "Public order",
name: "Violence and sexual offences (avg/yr)", "Public order offences (causing fear, alarm or distress) near the postcode"
bounds: Bounds::Percentile { ),
low: 2.0, ("Other crime", "Other crime (offences not classified elsewhere) near the postcode"),
high: 98.0,
},
step: 1.0,
description: "Average yearly violent and sexual offences in the area",
detail: "Average number of violence and sexual offences per year near the postcode, from police.uk street-level crime data. Includes assault, harassment, and sexual offences.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Burglary (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly burglary offences in the area",
detail: "Average number of burglary offences per year near the postcode, from police.uk street-level crime data. Includes residential and commercial burglary.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Robbery (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly robbery offences in the area",
detail: "Average number of robbery offences per year near the postcode, from police.uk street-level crime data. Robbery involves theft with force or threat of force.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Vehicle crime (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly vehicle crime in the area",
detail: "Average number of vehicle crime incidents per year near the postcode, from police.uk street-level crime data. Includes theft of and from vehicles.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Anti-social behaviour (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly anti-social behaviour incidents in the area",
detail: "Average number of anti-social behaviour incidents per year near the postcode, from police.uk street-level crime data. Includes nuisance, environmental, and personal anti-social behaviour.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Criminal damage and arson (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly criminal damage and arson in the area",
detail: "Average number of criminal damage and arson incidents per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Other theft (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly other theft offences in the area",
detail: "Average number of 'other theft' offences per year near the postcode, from police.uk street-level crime data. Includes theft not classified under burglary, vehicle crime, shoplifting, or bicycle theft.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Theft from the person (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly theft from the person in the area",
detail: "Average number of theft from the person offences per year near the postcode, from police.uk street-level crime data. Includes pickpocketing and bag snatching without force.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Shoplifting (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly shoplifting offences in the area",
detail: "Average number of shoplifting offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Bicycle theft (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly bicycle theft in the area",
detail: "Average number of bicycle theft offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Drugs (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly drug offences in the area",
detail: "Average number of drug offences per year near the postcode, from police.uk street-level crime data. Includes possession and trafficking offences.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Possession of weapons (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly weapons possession offences in the area",
detail: "Average number of possession of weapons offences per year near the postcode, from police.uk street-level crime data.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Public order (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly public order offences in the area",
detail: "Average number of public order offences per year near the postcode, from police.uk street-level crime data. Includes causing fear, alarm, or distress.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
Feature::Numeric(FeatureConfig {
name: "Other crime (avg/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 1.0,
description: "Average yearly other crime in the area",
detail: "Average number of other crime offences per year near the postcode, from police.uk street-level crime data. A catch-all category for offences not classified elsewhere.",
source: "crime",
prefix: "",
suffix: "",
raw: false,
absolute: false,
}),
], ],
}, },
FeatureGroup { FeatureGroup {
@ -826,9 +673,10 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
// shares sum to ~100% per neighbourhood (LSOA) and render as a // shares sum to ~100% per neighbourhood (LSOA) and render as a
// stacked composition (see STACKED_GROUPS["Neighbours"] in the // stacked composition (see STACKED_GROUPS["Neighbours"] in the
// frontend), like the ethnicity, qualifications and vote-share bars. // frontend), like the ethnicity, qualifications and vote-share bars.
// Unlike those, the three shares are ALSO offered as individual // Unlike those — each folded into a single dropdown filter that
// filters (they are not added to the display-only skip-list in // selects one band — the three tenure shares are offered as
// Filters.tsx), so users can target e.g. owner-occupier-heavy areas. // individual filters, so users can target e.g. owner-occupier-heavy
// areas.
Feature::Numeric(FeatureConfig { Feature::Numeric(FeatureConfig {
name: "% Owner occupied", name: "% Owner occupied",
bounds: Bounds::Fixed { min: 0.0, max: 100.0 }, bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
@ -1292,7 +1140,7 @@ mod tests {
"Income Score", // 0..100 percentile "Income Score", // 0..100 percentile
"% White", // 0..100 percentage "% White", // 0..100 percentage
"Noise (dB)", // 50..80, range > threshold "Noise (dB)", // 50..80, range > threshold
"Serious crime (avg/yr)", // Percentile bounds, fractional "Serious crime (/yr, 7y)", // Percentile bounds, fractional
"Interior height (m)", // step 0.1 "Interior height (m)", // step 0.1
"Estimated current price", // step 10000 "Estimated current price", // step 10000
] { ] {

View file

@ -334,6 +334,18 @@ struct Cli {
#[arg(long, env = "CRIME_BY_YEAR_PATH")] #[arg(long, env = "CRIME_BY_YEAR_PATH")]
crime_by_year_path: PathBuf, crime_by_year_path: PathBuf,
/// Path to the per-incident crime-records parquet (last 7 years, postcode-
/// sorted) backing the "individual crimes" list. Spilled to disk when
/// `--spill-dir` is set.
#[arg(long, env = "CRIME_RECORDS_PATH")]
crime_records_path: PathBuf,
/// Path to the precomputed national/per-outcode/per-sector crime-averages
/// parquet (built by pipeline.transform.area_crime_averages). The right pane
/// uses it to compare a selection's crime rates against its surroundings.
#[arg(long, env = "AREA_CRIME_AVERAGES_PATH")]
area_crime_averages_path: PathBuf,
/// Path to the per-unit-postcode population parquet (ONS Census 2021 usual /// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
/// residents; display-only side table for the right pane). Optional: when /// residents; display-only side table for the right pane). Optional: when
/// absent or missing, the area pane simply omits the population figure. /// absent or missing, the area pane simply omits the population figure.
@ -725,6 +737,16 @@ async fn main() -> anyhow::Result<()> {
Arc::new(data) Arc::new(data)
}; };
let crime_records = {
let path = &cli.crime_records_path;
if !path.exists() {
bail!("Crime-records parquet not found: {}", path.display());
}
let data = data::CrimeRecords::load(path, spill_dir)?;
trim_allocator("crime-records load");
Arc::new(data)
};
let population = match &cli.population_path { let population = match &cli.population_path {
Some(path) if path.exists() => { Some(path) if path.exists() => {
let data = data::PostcodePopulation::load(path)?; let data = data::PostcodePopulation::load(path)?;
@ -742,13 +764,11 @@ async fn main() -> anyhow::Result<()> {
}; };
let area_crime_averages = { let area_crime_averages = {
let data = property_data.compute_area_crime_averages(); let path = &cli.area_crime_averages_path;
info!( if !path.exists() {
outcodes = data.by_outcode.len(), bail!("Area-crime-averages parquet not found: {}", path.display());
sectors = data.by_sector.len(), }
crime_types = data.crime_types.len(), let data = data::AreaCrimeAverages::load(path)?;
"Per-outcode/sector crime averages computed"
);
trim_allocator("area crime averages"); trim_allocator("area crime averages");
Arc::new(data) Arc::new(data)
}; };
@ -781,6 +801,7 @@ async fn main() -> anyhow::Result<()> {
actual_listings, actual_listings,
developments, developments,
crime_by_year, crime_by_year,
crime_records,
population, population,
area_crime_averages, area_crime_averages,
token_cache, token_cache,
@ -899,6 +920,10 @@ async fn main() -> anyhow::Result<()> {
"/api/postcode-properties", "/api/postcode-properties",
get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)), get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
) )
.route(
"/api/crime-records",
get(routes::get_crime_records).layer(ConcurrencyLimitLayer::new(5)),
)
.route( .route(
"/api/screenshot", "/api/screenshot",
get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)), get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),

View file

@ -1319,6 +1319,49 @@ pub async fn ensure_collections(
ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?; ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?;
} }
// Per-user record of which property listings a user has opened, so visited listings
// can be drawn in a distinct colour on the map. One row per (user, url); the unique
// index makes re-clicks idempotent.
let clicked_listings_index =
"CREATE UNIQUE INDEX idx_clicked_listings_user_url ON clicked_listings (user, url)";
if !existing.iter().any(|n| n == "clicked_listings") {
let users_id = find_users_collection_id(client, base_url, &token).await?;
let user_only = Some("user = @request.auth.id".to_string());
create_collection(
client,
base_url,
&token,
CreateCollection {
name: "clicked_listings".to_string(),
r#type: "base".to_string(),
fields: vec![
Field::relation("user", &users_id),
Field::text("url", true),
Field::autodate("created", true, false),
Field::autodate("updated", true, true),
],
list_rule: user_only.clone(),
view_rule: user_only.clone(),
create_rule: user_only.clone(),
update_rule: user_only.clone(),
delete_rule: user_only,
indexes: vec![clicked_listings_index.to_string()],
},
)
.await?;
} else {
ensure_user_owned_rules(client, base_url, &token, "clicked_listings").await?;
ensure_autodate_fields(client, base_url, &token, "clicked_listings").await?;
ensure_collection_indexes(
client,
base_url,
&token,
"clicked_listings",
&[("idx_clicked_listings_user_url", clicked_listings_index)],
)
.await?;
}
Ok(()) Ok(())
} }

View file

@ -1,6 +1,7 @@
mod actual_listings; mod actual_listings;
mod ai_filters; mod ai_filters;
mod checkout; mod checkout;
mod crime_records;
mod developments; mod developments;
mod export; mod export;
mod features; mod features;
@ -35,6 +36,7 @@ pub(crate) mod travel_time;
pub use actual_listings::get_actual_listings; pub use actual_listings::get_actual_listings;
pub use ai_filters::{build_system_prompt, post_ai_filters}; pub use ai_filters::{build_system_prompt, post_ai_filters};
pub use checkout::post_checkout; pub use checkout::post_checkout;
pub use crime_records::get_crime_records;
pub use developments::get_developments; pub use developments::get_developments;
pub use export::get_export; pub use export::get_export;
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse}; pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};

View file

@ -336,8 +336,8 @@ mod tests {
"Good+ primary school catchments" "Good+ primary school catchments"
); );
assert_eq!( assert_eq!(
canonical_filter_name("Specific crimes:Burglary%20%28avg%2Fyr%29:1"), canonical_filter_name("Specific crimes:Burglary%20%28%2Fyr%2C%207y%29:1"),
"Burglary (avg/yr)" "Burglary (/yr, 7y)"
); );
assert_eq!( assert_eq!(
canonical_filter_name("Political vote share:%25%20Labour:0"), canonical_filter_name("Political vote share:%25%20Labour:0"),

View file

@ -25,9 +25,9 @@ pub fn build_system_prompt(
or \"max\" (at most this value). Never set two filters on the same feature.\n\ or \"max\" (at most this value). Never set two filters on the same feature.\n\
- Use EXACT feature names from the list spelling, capitalisation, and punctuation must match.\n\ - Use EXACT feature names from the list spelling, capitalisation, and punctuation must match.\n\
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\ - \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
- \"low crime\" / \"safe\" = low values on the Serious crime (avg/yr) and Minor crime (avg/yr) \ - \"low crime\" / \"safe\" = low values on the Serious crime (/yr, 7y) and Minor crime (/yr, 7y) \
features (area-normalised incident density near the postcode). Prefer these aggregates for broad \ features (average recorded incidents per year near the postcode, last 7 years). Prefer these aggregates for broad \
area safety; use specific crime features only when the user names a crime type.\n\ area safety; use specific crime features only when the user names a crime type. Use a \"(/yr, 2y)\" feature only when the user asks about recent crime.\n\
- \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \ - \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \
or low Distance to nearest park (km), depending on wording.\n\ or low Distance to nearest park (km), depending on wording.\n\
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\ - \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
@ -171,8 +171,8 @@ pub fn build_system_prompt(
parts.push( parts.push(
"\nUser: \"safe quiet area with good schools and parks\"\n\ "\nUser: \"safe quiet area with good schools and parks\"\n\
Output: {\"numeric_filters\": [\ Output: {\"numeric_filters\": [\
{\"name\": \"Serious crime (avg/yr)\", \"bound\": \"max\", \"value\": 5}, \ {\"name\": \"Serious crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 5}, \
{\"name\": \"Minor crime (avg/yr)\", \"bound\": \"max\", \"value\": 20}, \ {\"name\": \"Minor crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 20}, \
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \ {\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
{\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \ {\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \
{\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \ {\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \
@ -237,7 +237,7 @@ pub fn build_system_prompt(
"\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\ "\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\
Output: {\"numeric_filters\": [\ Output: {\"numeric_filters\": [\
{\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \ {\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \
{\"name\": \"Burglary (avg/yr)\", \"bound\": \"max\", \"value\": 10}, \ {\"name\": \"Burglary (/yr, 7y)\", \"bound\": \"max\", \"value\": 10}, \
{\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \ {\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}" \"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
.to_string(), .to_string(),

View file

@ -0,0 +1,194 @@
//! `GET /api/crime-records` — the individual police.uk crimes (last 7 years)
//! behind a selected hexagon or postcode, paginated. Display-only and
//! independent of the property filters, like the population figure: the records
//! are an attribute of the area, not of the filter-matching subset.
use std::str::FromStr;
use std::sync::Arc;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Json};
use axum::Extension;
use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize};
use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::licensing::{check_license_bounds, check_license_point, resolve_share_code};
use crate::parsing::{cell_for_row_cached, h3_cell_bounds, needs_parent, validate_h3_resolution};
use crate::state::SharedState;
use crate::utils::normalize_postcode;
/// Default and hard-cap page sizes for the records list.
const DEFAULT_LIMIT: usize = 200;
const MAX_LIMIT: usize = 500;
#[derive(Deserialize)]
pub struct CrimeRecordsParams {
/// Hexagon selection: H3 cell + resolution. Mutually exclusive with `postcode`.
pub h3: Option<String>,
pub resolution: Option<u8>,
/// Postcode selection.
pub postcode: Option<String>,
pub offset: Option<usize>,
pub limit: Option<usize>,
/// Lower bound on `month_index` (`year*12 + month0`) to restrict to a recent
/// window; omitted = all stored records (last 7 years).
pub since: Option<u32>,
/// Share-link code; grants scoped access for unlicensed users.
pub share: Option<String>,
}
#[derive(Serialize)]
pub struct CrimeRecord {
/// `"YYYY-MM"`.
pub month: String,
#[serde(rename = "type")]
pub crime_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub location: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub outcome: Option<String>,
pub lat: f32,
pub lon: f32,
}
#[derive(Serialize)]
pub struct CrimeRecordsResponse {
pub records: Vec<CrimeRecord>,
pub total: usize,
pub offset: usize,
pub truncated: bool,
}
fn format_month(month_index: u32) -> String {
let year = month_index / 12;
let month = month_index % 12 + 1;
format!("{year:04}-{month:02}")
}
pub async fn get_crime_records(
State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>,
Extension(geo): Extension<crate::demo_zone::DemoZone>,
Query(params): Query<CrimeRecordsParams>,
) -> Result<Json<CrimeRecordsResponse>, axum::response::Response> {
let state = shared.load_state();
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
let offset = params.offset.unwrap_or(0);
let limit = params.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
let since = params.since;
// Resolve the selection to a set of postcodes, after a license check scoped
// to the selection's geometry (bounds for a hexagon, point for a postcode).
enum Selection {
Hexagon { cell: u64, resolution: u8 },
Postcode(String),
}
let selection = if let Some(h3) = params.h3.clone() {
let cell = h3o::CellIndex::from_str(&h3).map_err(|error| {
warn!(h3 = %h3, error = %error, "Invalid H3 cell index");
(StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {error}")).into_response()
})?;
let resolution = params.resolution.ok_or_else(|| {
(StatusCode::BAD_REQUEST, "resolution is required with h3").into_response()
})?;
validate_h3_resolution(resolution).map_err(IntoResponse::into_response)?;
let bounds = h3_cell_bounds(cell, 0.0);
check_license_bounds(&user.0, bounds, geo.free_zone, share_bounds)?;
Selection::Hexagon {
cell: cell.into(),
resolution,
}
} else if let Some(postcode) = params.postcode.clone() {
let normalized = normalize_postcode(&postcode);
let &pc_idx = state
.postcode_data
.postcode_to_idx
.get(&normalized)
.ok_or_else(|| {
(StatusCode::NOT_FOUND, format!("Postcode not found: {normalized}")).into_response()
})?;
let (lat, lon) = state.postcode_data.centroids[pc_idx];
check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
Selection::Postcode(normalized)
} else {
return Err((StatusCode::BAD_REQUEST, "h3 or postcode is required").into_response());
};
let result = tokio::task::spawn_blocking(move || -> Result<CrimeRecordsResponse, String> {
// Distinct postcodes covered by the selection.
let postcodes: Vec<String> = match selection {
Selection::Postcode(pc) => vec![pc],
Selection::Hexagon { cell, resolution } => {
let h3_res = h3o::Resolution::try_from(resolution)
.map_err(|err| format!("Invalid H3 resolution {resolution}: {err}"))?;
let need_parent = needs_parent(resolution);
let h3o_cell = h3o::CellIndex::try_from(cell)
.map_err(|err| format!("Invalid H3 cell: {err}"))?;
let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(h3o_cell, 0.001);
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
let mut seen: FxHashSet<&str> = FxHashSet::default();
let mut out: Vec<String> = Vec::new();
state.grid.for_each_in_bounds(
min_lat,
min_lon,
max_lat,
max_lon,
|row_idx| {
let row = row_idx as usize;
if cell_for_row_cached(
row,
&state.h3_cells,
h3_res,
need_parent,
&mut h3_cache,
) == cell
{
let pc = state.data.postcode(row);
if seen.insert(pc) {
out.push(pc.to_string());
}
}
},
);
out
}
};
let pc_refs: Vec<&str> = postcodes.iter().map(String::as_str).collect();
let indices = state.crime_records.gather(&pc_refs, since);
let total = indices.len();
let records: Vec<CrimeRecord> = indices
.iter()
.skip(offset)
.take(limit)
.map(|&idx| {
let v = state.crime_records.view(idx);
CrimeRecord {
month: format_month(v.month_index),
crime_type: v.crime_type.to_string(),
location: v.location.map(str::to_string),
outcome: v.outcome.map(str::to_string),
lat: v.lat,
lon: v.lon,
}
})
.collect();
let truncated = offset + records.len() < total;
Ok(CrimeRecordsResponse {
records,
total,
offset,
truncated,
})
})
.await
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response())?
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error).into_response())?;
info!(total = result.total, returned = result.records.len(), "GET /api/crime-records");
Ok(Json(result))
}

View file

@ -74,20 +74,20 @@ pub struct CrimeYearPoint {
#[derive(Serialize)] #[derive(Serialize)]
pub struct CrimeYearStats { pub struct CrimeYearStats {
/// Underlying crime type (e.g. "Burglary"). Matches existing crime feature /// Underlying crime type, bare (e.g. "Burglary"). Matches the type prefix of
/// names with the `" (avg/yr)"` suffix stripped. /// the `"(/yr, …)"` crime features.
pub name: String, pub name: String,
pub points: Vec<CrimeYearPoint>, pub points: Vec<CrimeYearPoint>,
} }
/// Average headline crime rate (avg/yr) for one crime type across the /// Average crime count for one crime feature across the selection's outcode and
/// selection's outcode and postcode sector. Comparable to the national average /// postcode sector. Comparable to the national average shown per metric in the
/// shown per metric in the right pane. /// right pane.
#[derive(Serialize)] #[derive(Serialize)]
pub struct CrimeAreaAverage { pub struct CrimeAreaAverage {
/// Crime type, bare (e.g. "Burglary"). Matches `CrimeYearStats.name`. /// Full crime-feature name (e.g. "Burglary (/yr, 7y)").
pub name: String, pub name: String,
/// Exact national mean (avg/yr) — the frontend prefers this over the /// Exact national mean count — the frontend prefers this over the
/// histogram-bin national average for crime so all four numbers in the row /// histogram-bin national average for crime so all four numbers in the row
/// share one estimator. /// share one estimator.
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
@ -161,10 +161,15 @@ pub struct HexagonStatsResponse {
/// present only when sector crime averages are available for it. /// present only when sector crime averages are available for it.
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub crime_sector: Option<String>, pub crime_sector: Option<String>,
/// Per-crime-type average rates across the central postcode's outcode and /// Per-crime-type average counts across the central postcode's outcode and
/// sector, shown alongside the national average for each crime metric. /// sector, shown alongside the national average for each crime metric.
#[serde(skip_serializing_if = "Vec::is_empty")] #[serde(skip_serializing_if = "Vec::is_empty")]
pub crime_area_averages: Vec<CrimeAreaAverage>, pub crime_area_averages: Vec<CrimeAreaAverage>,
/// Total individual crime records (last 7 years) across the distinct
/// postcodes in this selection — the count behind the "individual crimes"
/// list. Filter-independent, like `population`.
#[serde(skip_serializing_if = "Option::is_none")]
pub crime_total_records: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub central_postcode: Option<String>, pub central_postcode: Option<String>,
/// Total usual residents (ONS Census 2021) living across the distinct /// Total usual residents (ONS Census 2021) living across the distinct
@ -699,26 +704,42 @@ pub async fn get_hexagon_stats(
&field_set, &field_set,
); );
// Sum usual residents across the distinct postcodes covered by the // Distinct postcodes covered by the hexagon, taken over `area_rows` (all
// hexagon. Computed over `area_rows` (all properties in the cell), not // properties in the cell), not the filter-matching subset — population and
// the filter-matching subset, so toggling filters never changes it — // the crime-records count are attributes of the area, independent of the
// population is an attribute of the area, like the council-house count. // active filters (like the council-house count).
let mut seen: HashSet<&str> = HashSet::new();
let mut area_postcodes: Vec<&str> = Vec::new();
for &row in &area_rows {
let pc = state.data.postcode(row);
if seen.insert(pc) {
area_postcodes.push(pc);
}
}
let population = { let population = {
let mut seen: HashSet<&str> = HashSet::new();
let mut total: u64 = 0; let mut total: u64 = 0;
let mut found = false; let mut found = false;
for &row in &area_rows { for &pc in &area_postcodes {
let pc = state.data.postcode(row); if let Some(p) = state.population.for_postcode(pc) {
if seen.insert(pc) { total += p as u64;
if let Some(p) = state.population.for_postcode(pc) { found = true;
total += p as u64;
found = true;
}
} }
} }
found.then(|| total.min(u32::MAX as u64) as u32) found.then(|| total.min(u32::MAX as u64) as u32)
}; };
let crime_total_records = {
// Sum the per-postcode counts straight from the CSR index instead of
// materializing (and sorting) every record index: this keeps the
// mmap-backed columns cold on the hot hexagon path.
let total: u64 = area_postcodes
.iter()
.map(|pc| state.crime_records.total_for(pc) as u64)
.sum();
(total > 0).then(|| total.min(u32::MAX as u64) as u32)
};
Ok(HexagonStatsResponse { Ok(HexagonStatsResponse {
count: total_count, count: total_count,
numeric_features, numeric_features,
@ -729,6 +750,7 @@ pub async fn get_hexagon_stats(
crime_outcode, crime_outcode,
crime_sector, crime_sector,
crime_area_averages, crime_area_averages,
crime_total_records,
central_postcode, central_postcode,
population, population,
filter_exclusions, filter_exclusions,

View file

@ -227,6 +227,11 @@ pub async fn get_postcode_stats(
// Usual residents (Census 2021) for this postcode. Display-only. // Usual residents (Census 2021) for this postcode. Display-only.
let population = state.population.for_postcode(&postcode_str); let population = state.population.for_postcode(&postcode_str);
let crime_total_records = {
let total = state.crime_records.total_for(&postcode_str);
(total > 0).then_some(total)
};
Ok(HexagonStatsResponse { Ok(HexagonStatsResponse {
count: total_count, count: total_count,
numeric_features, numeric_features,
@ -237,6 +242,7 @@ pub async fn get_postcode_stats(
crime_outcode, crime_outcode,
crime_sector, crime_sector,
crime_area_averages, crime_area_averages,
crime_total_records,
central_postcode: None, central_postcode: None,
population, population,
filter_exclusions, filter_exclusions,

View file

@ -127,15 +127,20 @@ fn is_allowed_param_key(key: &str) -> bool {
| "filter" | "filter"
| "school" | "school"
| "crime" | "crime"
| "crimeSeverity"
| "voteShare" | "voteShare"
| "ethnicity" | "ethnicity"
| "qualification"
| "tenure"
| "amenityDistance" | "amenityDistance"
| "transportDistance" | "transportDistance"
| "amenityCount2km" | "amenityCount2km"
| "amenityCount5km" | "amenityCount5km"
| "poi" | "poi"
| "overlay" | "overlay"
| "crimeType"
| "basemap" | "basemap"
| "colorOpacity"
| "tab" | "tab"
| "pc" | "pc"
| "tt" | "tt"
@ -594,6 +599,22 @@ mod tests {
assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite"); assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
} }
#[test]
fn preserves_all_filter_params_for_share_links() {
// Every filter param emitted by the frontend's stateToParams() must survive
// shortening; an unsupported key is rejected outright (see is_allowed_param_key),
// which fails the whole share link rather than dropping a single filter.
// Values use %3A (":") since form re-serialization keeps it stable.
let query = "crimeSeverity=Serious%3A0%3A5\
&qualification=Degree%3A20%3A80\
&tenure=Owner%3A30%3A90\
&crimeType=burglary\
&colorOpacity=60";
let params = sanitized_query_params(query, false).unwrap();
assert_eq!(params, query);
}
#[test] #[test]
fn escapes_html_attributes() { fn escapes_html_attributes() {
assert_eq!(escape_attr(r#""'><&"#), "&quot;&#39;&gt;&lt;&amp;"); assert_eq!(escape_attr(r#""'><&"#), "&quot;&#39;&gt;&lt;&amp;");

View file

@ -11,8 +11,8 @@ use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
use crate::utils::{postcode_outcode, postcode_sector}; use crate::utils::{postcode_outcode, postcode_sector};
use super::hexagon_stats::{ use super::hexagon_stats::{
CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats, CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats,
NumericFeatureStats, PricePoint, HistogramStats, NumericFeatureStats, PricePoint,
}; };
/// Extract price history (year, price) pairs from matching rows, downsampled if needed. /// Extract price history (year, price) pairs from matching rows, downsampled if needed.
@ -352,11 +352,14 @@ pub fn compute_crime_by_year(
let mut out = Vec::new(); let mut out = Vec::new();
for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() { for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() {
// Crime types in the by-year side table are bare (e.g. "Burglary"), while // Crime types in the by-year side table are bare (e.g. "Burglary"), while
// the configured feature names carry an " (avg/yr)" suffix. Match either // the configured feature names carry a window suffix ("Burglary (/yr,
// form so callers can pass the feature names they already know. // 7y)"). Emit the bare-type trend if the bare name is requested directly or
// any of its windowed features is.
if fields_specified { if fields_specified {
let with_suffix = format!("{name} (avg/yr)"); let prefix = format!("{name} (");
if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) { if !field_set.contains(name.as_str())
&& !field_set.iter().any(|f| f.starts_with(&prefix))
{
continue; continue;
} }
} }
@ -395,6 +398,7 @@ pub fn compute_crime_by_year(
out out
} }
/// Latest year present anywhere in the by-year crime dataset. The client /// Latest year present anywhere in the by-year crime dataset. The client
/// compares each selection's last charted year against this to caption /// compares each selection's last charted year against this to caption
/// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as /// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as
@ -440,13 +444,10 @@ pub fn area_crime_averages_for(
let mut out = Vec::new(); let mut out = Vec::new();
for (idx, name) in averages.crime_types.iter().enumerate() { for (idx, name) in averages.crime_types.iter().enumerate() {
// Crime types are bare here ("Burglary"); requested fields may carry the // `name` is the full crime-feature name here (e.g. "Burglary (/yr,
// " (avg/yr)" suffix — accept either form, like compute_crime_by_year. // 7y)"), matching exactly the feature fields the caller requests.
if fields_specified { if fields_specified && !field_set.contains(name.as_str()) {
let with_suffix = format!("{name} (avg/yr)"); continue;
if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
continue;
}
} }
let national_val = finite_at(Some(&averages.national), idx); let national_val = finite_at(Some(&averages.national), idx);
let outcode_val = finite_at(outcode_means, idx); let outcode_val = finite_at(outcode_means, idx);
@ -595,7 +596,10 @@ mod tests {
let mut by_sector = rustc_hash::FxHashMap::default(); let mut by_sector = rustc_hash::FxHashMap::default();
by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]); by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
AreaCrimeAverages { AreaCrimeAverages {
crime_types: vec!["Burglary".to_string(), "Robbery".to_string()], crime_types: vec![
"Burglary (/yr, 7y)".to_string(),
"Robbery (/yr, 7y)".to_string(),
],
national: vec![8.0, 6.0], national: vec![8.0, 6.0],
by_outcode, by_outcode,
by_sector, by_sector,
@ -611,12 +615,18 @@ mod tests {
assert_eq!(sector.as_deref(), Some("E14 2")); assert_eq!(sector.as_deref(), Some("E14 2"));
assert_eq!(out.len(), 2); assert_eq!(out.len(), 2);
let burglary = out.iter().find(|c| c.name == "Burglary").unwrap(); let burglary = out
.iter()
.find(|c| c.name == "Burglary (/yr, 7y)")
.unwrap();
assert_eq!(burglary.national, Some(8.0)); assert_eq!(burglary.national, Some(8.0));
assert_eq!(burglary.outcode, Some(10.0)); assert_eq!(burglary.outcode, Some(10.0));
assert_eq!(burglary.sector, Some(5.0)); assert_eq!(burglary.sector, Some(5.0));
let robbery = out.iter().find(|c| c.name == "Robbery").unwrap(); let robbery = out
.iter()
.find(|c| c.name == "Robbery (/yr, 7y)")
.unwrap();
assert_eq!(robbery.national, Some(6.0)); assert_eq!(robbery.national, Some(6.0));
// The outcode value was NaN — dropped to None; the sector value is finite. // The outcode value was NaN — dropped to None; the sector value is finite.
assert_eq!(robbery.outcode, None); assert_eq!(robbery.outcode, None);
@ -626,11 +636,13 @@ mod tests {
#[test] #[test]
fn area_crime_averages_respect_fields_filter() { fn area_crime_averages_respect_fields_filter() {
let avgs = sample_averages(); let avgs = sample_averages();
// The suffixed feature-name form is accepted, like compute_crime_by_year. // Area averages are keyed by the full crime-feature name.
let fields: HashSet<String> = ["Burglary (avg/yr)".to_string()].into_iter().collect(); let fields: HashSet<String> = ["Burglary (/yr, 7y)".to_string()]
.into_iter()
.collect();
let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields); let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
assert_eq!(out.len(), 1); assert_eq!(out.len(), 1);
assert_eq!(out[0].name, "Burglary"); assert_eq!(out[0].name, "Burglary (/yr, 7y)");
} }
#[test] #[test]

View file

@ -6,9 +6,9 @@ use rustc_hash::FxHashMap;
use crate::auth::TokenCache; use crate::auth::TokenCache;
use crate::bugsink::FrontendConfig as BugsinkFrontendConfig; use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
use crate::data::{ use crate::data::{
ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData, ActualListingData, AreaCrimeAverages, CrimeByYearData, CrimeRecords,
POICategoryGroup, POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData, DevelopmentData, OutcodeData, POICategoryGroup, POIData, PlaceData, PostcodeData,
TravelTimeStore, PostcodePopulation, PropertyData, TravelTimeStore,
}; };
use crate::licensing::ShareBoundsCache; use crate::licensing::ShareBoundsCache;
use crate::pocketbase::SuperuserTokenCache; use crate::pocketbase::SuperuserTokenCache;
@ -52,10 +52,13 @@ pub struct AppState {
pub developments: Arc<DevelopmentData>, pub developments: Arc<DevelopmentData>,
/// Per-LSOA per-year crime counts used by the right pane to plot trends. /// Per-LSOA per-year crime counts used by the right pane to plot trends.
pub crime_by_year: Arc<CrimeByYearData>, pub crime_by_year: Arc<CrimeByYearData>,
/// Per-postcode individual crime records (last 7 years), spill-backed,
/// served by the `/api/crime-records` endpoint and counted in stats.
pub crime_records: Arc<CrimeRecords>,
/// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the /// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
/// right pane. Display-only — never filterable. Empty when no data is loaded. /// right pane. Display-only — never filterable. Empty when no data is loaded.
pub population: Arc<PostcodePopulation>, pub population: Arc<PostcodePopulation>,
/// Precomputed per-outcode and per-postcode-sector average crime rates, /// Precomputed per-outcode and per-postcode-sector average crime counts,
/// shown in the right pane alongside the national average for each metric. /// shown in the right pane alongside the national average for each metric.
pub area_crime_averages: Arc<AreaCrimeAverages>, pub area_crime_averages: Arc<AreaCrimeAverages>,
/// Token validation cache (60s TTL) /// Token validation cache (60s TTL)
@ -178,6 +181,7 @@ impl AppState {
series_by_postcode: FxHashMap::default(), series_by_postcode: FxHashMap::default(),
covered_years_by_postcode: FxHashMap::default(), covered_years_by_postcode: FxHashMap::default(),
}), }),
crime_records: Arc::new(CrimeRecords::empty()),
population: Arc::new(PostcodePopulation::empty()), population: Arc::new(PostcodePopulation::empty()),
area_crime_averages: Arc::new(AreaCrimeAverages::empty()), area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
token_cache: Arc::new(TokenCache::new()), token_cache: Arc::new(TokenCache::new()),