lgtm
This commit is contained in:
parent
2efa4d9f47
commit
5e73287eaf
99 changed files with 6392 additions and 1462 deletions
|
|
@ -56,4 +56,4 @@ EXPOSE 8001
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
|
||||||
CMD curl -f http://localhost:8001/health || exit 1
|
CMD curl -f http://localhost:8001/health || exit 1
|
||||||
ENTRYPOINT ["./property-map-server"]
|
ENTRYPOINT ["./property-map-server"]
|
||||||
CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--satellite-tiles", "/app/data/satellite.pmtiles", "--satellite-highres-tiles", "/app/data/satellite_highres.pmtiles", "--noise-overlay-tiles", "/app/data/noise_lden_10m.pmtiles", "--crime-hotspot-tiles", "/app/data/crime_hotspots.pmtiles", "--tree-overlay-tiles", "/app/data/trees_outside_woodlands.pmtiles", "--property-border-tiles", "/app/data/property_borders.pmtiles", "--crime-by-year-path", "/app/data/crime_by_postcode_by_year.parquet", "--area-crime-averages-path", "/app/data/area_crime_averages.parquet", "--population-path", "/app/data/population_by_postcode.parquet", "--developments-path", "/app/data/development_sites.parquet", "--dist", "/app/frontend/dist"]
|
CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--satellite-tiles", "/app/data/satellite.pmtiles", "--satellite-highres-tiles", "/app/data/satellite_highres.pmtiles", "--noise-overlay-tiles", "/app/data/noise_lden_10m.pmtiles", "--crime-hotspot-tiles", "/app/data/crime_hotspots.pmtiles", "--tree-overlay-tiles", "/app/data/trees_outside_woodlands.pmtiles", "--property-border-tiles", "/app/data/property_borders.pmtiles", "--crime-by-year-path", "/app/data/crime_by_postcode_by_year.parquet", "--crime-records-path", "/app/data/crime_records.parquet", "--area-crime-averages-path", "/app/data/area_crime_averages.parquet", "--population-path", "/app/data/population_by_postcode.parquet", "--developments-path", "/app/data/development_sites.parquet", "--dist", "/app/frontend/dist"]
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,10 @@ POSTCODES_RAW := $(DATA_DIR)/gb-postcodes-v5
|
||||||
POSTCODES_PQ := $(DATA_DIR)/postcode.parquet
|
POSTCODES_PQ := $(DATA_DIR)/postcode.parquet
|
||||||
PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
|
PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
|
||||||
MERGE_STAMP := $(DATA_DIR)/.merge_done
|
MERGE_STAMP := $(DATA_DIR)/.merge_done
|
||||||
|
PRICE_INPUTS := $(DATA_DIR)/price_inputs.parquet
|
||||||
|
POSTCODE_CENTROIDS := $(DATA_DIR)/postcode_centroids.parquet
|
||||||
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
|
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
|
||||||
|
PRICE_ESTIMATES := $(DATA_DIR)/price_estimates.parquet
|
||||||
PRICES_STAMP := $(DATA_DIR)/.prices_done
|
PRICES_STAMP := $(DATA_DIR)/.prices_done
|
||||||
EPC := $(MANUAL_DATA)/domestic-csv.zip
|
EPC := $(MANUAL_DATA)/domestic-csv.zip
|
||||||
ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet
|
ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet
|
||||||
|
|
@ -38,6 +41,8 @@ TENURE := $(DATA_DIR)/tenure_by_lsoa.parquet
|
||||||
CRIME_DIR := $(DATA_DIR)/crime
|
CRIME_DIR := $(DATA_DIR)/crime
|
||||||
CRIME := $(DATA_DIR)/crime_by_postcode.parquet
|
CRIME := $(DATA_DIR)/crime_by_postcode.parquet
|
||||||
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet
|
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet
|
||||||
|
CRIME_RECORDS := $(DATA_DIR)/crime_records.parquet
|
||||||
|
AREA_CRIME_AVERAGES := $(DATA_DIR)/area_crime_averages.parquet
|
||||||
POPULATION := $(DATA_DIR)/population_by_postcode.parquet
|
POPULATION := $(DATA_DIR)/population_by_postcode.parquet
|
||||||
CRIME_STAMP := $(CRIME_DIR)/.downloaded
|
CRIME_STAMP := $(CRIME_DIR)/.downloaded
|
||||||
NOISE := $(DATA_DIR)/road_noise.parquet
|
NOISE := $(DATA_DIR)/road_noise.parquet
|
||||||
|
|
@ -91,8 +96,11 @@ VALIDATE_OUTPUTS := uv run python -m pipeline.validate_outputs
|
||||||
|
|
||||||
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
|
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
|
||||||
MERGE_DEPS := pipeline/transform/merge.py
|
MERGE_DEPS := pipeline/transform/merge.py
|
||||||
|
AREA_CRIME_AVERAGES_DEPS := pipeline/transform/area_crime_averages.py
|
||||||
PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py
|
PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py
|
||||||
PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py
|
PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py
|
||||||
|
PRICE_JOIN_DEPS := pipeline/transform/join_price_estimates.py pipeline/transform/price_estimation/utils.py
|
||||||
|
PRICE_INPUTS_DEPS := pipeline/transform/property_base.py pipeline/utils/postcode_mapping.py
|
||||||
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
|
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
|
||||||
PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \
|
PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \
|
||||||
pipeline/transform/postcode_boundaries/greenspace.py \
|
pipeline/transform/postcode_boundaries/greenspace.py \
|
||||||
|
|
@ -116,11 +124,11 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
|
||||||
download-postcodes download-noise download-inspire download-crime \
|
download-postcodes download-noise download-inspire download-crime \
|
||||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \
|
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \
|
||||||
download-map-assets \
|
download-map-assets \
|
||||||
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
transform-pois transform-epc-pp transform-crime transform-poi-proximity transform-area-crime-averages \
|
||||||
transform-school-catchments transform-tree-density \
|
transform-school-catchments transform-tree-density \
|
||||||
generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites
|
generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites
|
||||||
|
|
||||||
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) $(CRIME_RECORDS) $(AREA_CRIME_AVERAGES) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
||||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
|
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
|
||||||
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
|
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
|
||||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
|
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
|
||||||
|
|
@ -178,6 +186,7 @@ transform-pois: $(POIS_FILTERED)
|
||||||
transform-epc-pp: $(EPC_PP)
|
transform-epc-pp: $(EPC_PP)
|
||||||
transform-crime: $(CRIME)
|
transform-crime: $(CRIME)
|
||||||
transform-poi-proximity: $(POI_PROXIMITY)
|
transform-poi-proximity: $(POI_PROXIMITY)
|
||||||
|
transform-area-crime-averages: $(AREA_CRIME_AVERAGES)
|
||||||
transform-school-catchments: $(SCHOOL_CATCH)
|
transform-school-catchments: $(SCHOOL_CATCH)
|
||||||
transform-tree-density: $(TREE_DENSITY_PC)
|
transform-tree-density: $(TREE_DENSITY_PC)
|
||||||
generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP)
|
generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP)
|
||||||
|
|
@ -383,9 +392,9 @@ $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(GIAS) $(OFSTE
|
||||||
$(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
|
$(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
|
||||||
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
||||||
|
|
||||||
$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
|
$(CRIME) $(CRIME_BY_YEAR) $(CRIME_RECORDS) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
|
||||||
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
|
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
|
||||||
uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR)
|
uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR) --output-records $(CRIME_RECORDS)
|
||||||
|
|
||||||
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
|
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
|
||||||
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
|
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
|
||||||
|
|
@ -450,16 +459,42 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||||
$(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP)
|
$(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP)
|
||||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
|
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
|
||||||
|
|
||||||
$(PRICE_INDEX): $(MERGE_STAMP) $(PRICE_INDEX_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
|
# Slim price-estimation inputs, built straight from epc_pp + arcgis (NOT the
|
||||||
uv run python -m pipeline.transform.price_estimation.index --input $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --output $@
|
# merge outputs). This is what decouples the expensive index/kNN from merge:
|
||||||
|
# adding an area or property column to merge does not change these files, so the
|
||||||
|
# price index and estimates are reused and only the cheap join re-runs.
|
||||||
|
$(PRICE_INPUTS) $(POSTCODE_CENTROIDS) &: $(EPC_PP) $(ARCGIS) $(PRICE_INPUTS_DEPS)
|
||||||
|
uv run python -m pipeline.transform.property_base --epc-pp $(EPC_PP) --arcgis $(ARCGIS) --output $(PRICE_INPUTS) --centroids $(POSTCODE_CENTROIDS)
|
||||||
|
$(VALIDATE_OUTPUTS) --parquet $(PRICE_INPUTS) --parquet $(POSTCODE_CENTROIDS)
|
||||||
|
|
||||||
|
$(PRICE_INDEX): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX_DEPS)
|
||||||
|
uv run python -m pipeline.transform.price_estimation.index --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --output $@
|
||||||
$(VALIDATE_OUTPUTS) --parquet $@
|
$(VALIDATE_OUTPUTS) --parquet $@
|
||||||
|
|
||||||
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
|
# Estimate prices on the slim inputs and write a standalone price_estimates.parquet
|
||||||
|
# (natural key + the two estimate columns). Never touches properties.parquet.
|
||||||
|
$(PRICE_ESTIMATES): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS)
|
||||||
|
uv run python -m pipeline.transform.price_estimation.estimate --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --index $(PRICE_INDEX) --output $@
|
||||||
|
$(VALIDATE_OUTPUTS) --parquet $@
|
||||||
|
|
||||||
|
# Join the estimate columns back onto properties.parquet by natural key
|
||||||
|
# (idempotent, atomic). Re-runs whenever merge or the estimates change.
|
||||||
|
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_ESTIMATES) $(PRICE_JOIN_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ) $(PRICE_INDEX)
|
||||||
@rm -f $@
|
@rm -f $@
|
||||||
uv run python -m pipeline.transform.price_estimation.estimate --properties $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --index $(PRICE_INDEX)
|
uv run python -m pipeline.transform.join_price_estimates --properties $(PROPERTIES_PQ) --estimates $(PRICE_ESTIMATES)
|
||||||
$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
|
$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
|
||||||
@touch $@
|
@touch $@
|
||||||
|
|
||||||
|
# ── Area crime averages (post-merge) ────────────────────
|
||||||
|
# National / per-outcode / per-sector property-weighted mean headline crime
|
||||||
|
# rates for the right pane. Depends on $(PRICES_STAMP) so it reads the
|
||||||
|
# finalised properties.parquet (the price-estimate join rewrites it); only the
|
||||||
|
# per-postcode property counts and the crime values from postcode.parquet are
|
||||||
|
# used, both unaffected by that join.
|
||||||
|
$(AREA_CRIME_AVERAGES): $(PRICES_STAMP) $(AREA_CRIME_AVERAGES_DEPS) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
|
||||||
|
uv run python -m pipeline.transform.area_crime_averages --postcodes $(POSTCODES_PQ) --properties $(PROPERTIES_PQ) --output $@
|
||||||
|
$(VALIDATE_OUTPUTS) --parquet $@
|
||||||
|
|
||||||
$(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
|
$(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
|
||||||
$(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
$(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||||
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \
|
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ services:
|
||||||
command: >
|
command: >
|
||||||
bash -c "
|
bash -c "
|
||||||
cargo install cargo-watch &&
|
cargo install cargo-watch &&
|
||||||
cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data/properties.parquet --postcode-features /app/property-data/postcode.parquet --pois /app/property-data/filtered_uk_pois.parquet --places /app/property-data/places.parquet --tiles /app/property-data/uk.pmtiles --postcodes /app/property-data/postcode_boundaries --travel-times /app/property-data/travel-times --satellite-tiles /app/property-data/satellite.pmtiles --satellite-highres-tiles /app/property-data/satellite_highres.pmtiles --noise-overlay-tiles /app/property-data/noise_lden_10m.pmtiles --crime-hotspot-tiles /app/property-data/crime_hotspots.pmtiles --tree-overlay-tiles /app/property-data/trees_outside_woodlands.pmtiles --property-border-tiles /app/property-data/property_borders.pmtiles --crime-by-year-path /app/property-data/crime_by_postcode_by_year.parquet --population-path /app/property-data/population_by_postcode.parquet'
|
cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data/properties.parquet --postcode-features /app/property-data/postcode.parquet --pois /app/property-data/filtered_uk_pois.parquet --places /app/property-data/places.parquet --tiles /app/property-data/uk.pmtiles --postcodes /app/property-data/postcode_boundaries --travel-times /app/property-data/travel-times --satellite-tiles /app/property-data/satellite.pmtiles --satellite-highres-tiles /app/property-data/satellite_highres.pmtiles --noise-overlay-tiles /app/property-data/noise_lden_10m.pmtiles --crime-hotspot-tiles /app/property-data/crime_hotspots.pmtiles --tree-overlay-tiles /app/property-data/trees_outside_woodlands.pmtiles --property-border-tiles /app/property-data/property_borders.pmtiles --crime-by-year-path /app/property-data/crime_by_postcode_by_year.parquet --crime-records-path /app/property-data/crime_records.parquet --area-crime-averages-path /app/property-data/area_crime_averages.parquet --population-path /app/property-data/population_by_postcode.parquet'
|
||||||
"
|
"
|
||||||
ports:
|
ports:
|
||||||
- "8001:8001"
|
- "8001:8001"
|
||||||
|
|
|
||||||
|
|
@ -383,7 +383,7 @@ export function SavedPage({
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
if (!cancelled) {
|
if (!cancelled) {
|
||||||
setShareLinksError(err instanceof Error ? err.message : 'Failed to fetch share links');
|
setShareLinksError(err instanceof Error ? err.message : t('accountPage.fetchShareLinksError'));
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.finally(() => {
|
.finally(() => {
|
||||||
|
|
@ -393,7 +393,7 @@ export function SavedPage({
|
||||||
return () => {
|
return () => {
|
||||||
cancelled = true;
|
cancelled = true;
|
||||||
};
|
};
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const tabClass = (tab: string) =>
|
const tabClass = (tab: string) =>
|
||||||
`px-4 py-2 text-sm font-medium border-b-2 transition-colors ${
|
`px-4 py-2 text-sm font-medium border-b-2 transition-colors ${
|
||||||
|
|
@ -738,7 +738,7 @@ function InviteSection({ user }: { user: AuthUser }) {
|
||||||
setInviteUrl((prev) => ({ ...prev, [type]: data.url }));
|
setInviteUrl((prev) => ({ ...prev, [type]: data.url }));
|
||||||
fetchInviteHistory();
|
fetchInviteHistory();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Failed to create invite';
|
const msg = err instanceof Error ? err.message : t('invitesPage.createInviteError');
|
||||||
setInviteError((prev) => ({ ...prev, [type]: msg }));
|
setInviteError((prev) => ({ ...prev, [type]: msg }));
|
||||||
} finally {
|
} finally {
|
||||||
setCreatingInvite((prev) => ({ ...prev, [type]: false }));
|
setCreatingInvite((prev) => ({ ...prev, [type]: false }));
|
||||||
|
|
@ -931,7 +931,7 @@ export default function AccountPage({
|
||||||
assertOk(res, 'Update newsletter');
|
assertOk(res, 'Update newsletter');
|
||||||
await onRefreshAuth();
|
await onRefreshAuth();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Failed to update newsletter';
|
const msg = err instanceof Error ? err.message : t('accountPage.updateNewsletterError');
|
||||||
setNewsletterError(msg);
|
setNewsletterError(msg);
|
||||||
} finally {
|
} finally {
|
||||||
setNewsletterSaving(false);
|
setNewsletterSaving(false);
|
||||||
|
|
|
||||||
|
|
@ -150,7 +150,7 @@ function ProductDemoVideo() {
|
||||||
<track
|
<track
|
||||||
kind="captions"
|
kind="captions"
|
||||||
srcLang={(i18n.language ?? 'en').split('-')[0]}
|
srcLang={(i18n.language ?? 'en').split('-')[0]}
|
||||||
label="Captions"
|
label={t('common.captions')}
|
||||||
src={`/video/${productDemoSlug}.vtt`}
|
src={`/video/${productDemoSlug}.vtt`}
|
||||||
/>
|
/>
|
||||||
</video>
|
</video>
|
||||||
|
|
|
||||||
|
|
@ -74,7 +74,7 @@ const DEMO_FEATURES: FeatureMeta[] = [
|
||||||
prefix: '£',
|
prefix: '£',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'Serious crime (avg/yr)',
|
name: 'Serious crime (/yr, 7y)',
|
||||||
type: 'numeric',
|
type: 'numeric',
|
||||||
group: 'Crime',
|
group: 'Crime',
|
||||||
min: 0,
|
min: 0,
|
||||||
|
|
@ -763,7 +763,7 @@ function RightPaneOnlyScreen({
|
||||||
<span className="truncate">{t('home.showcasePoliticalVoteShare')}</span>
|
<span className="truncate">{t('home.showcasePoliticalVoteShare')}</span>
|
||||||
</div>
|
</div>
|
||||||
<span className="shrink-0 text-xs font-black text-teal-700 dark:text-teal-300">
|
<span className="shrink-0 text-xs font-black text-teal-700 dark:text-teal-300">
|
||||||
2024 GE
|
{t('home.showcaseGe2024')}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<StackedBarChart
|
<StackedBarChart
|
||||||
|
|
|
||||||
|
|
@ -155,11 +155,11 @@ export default function InvitePage({
|
||||||
window.location.href = data.checkout_url;
|
window.location.href = data.checkout_url;
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : 'Failed to redeem invite');
|
setError(err instanceof Error ? err.message : t('invitePage.redeemFailed'));
|
||||||
} finally {
|
} finally {
|
||||||
setRedeeming(false);
|
setRedeeming(false);
|
||||||
}
|
}
|
||||||
}, [code, user, onLicenseGranted]);
|
}, [code, user, onLicenseGranted, t]);
|
||||||
|
|
||||||
if (screenshotMode && loading) {
|
if (screenshotMode && loading) {
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -262,7 +262,7 @@ function SocialVideoCard({
|
||||||
onPause={() => setIsPlaying(false)}
|
onPause={() => setIsPlaying(false)}
|
||||||
onEnded={() => setIsPlaying(false)}
|
onEnded={() => setIsPlaying(false)}
|
||||||
>
|
>
|
||||||
<track kind="captions" srcLang="en" label="Captions" src={`/video/${slug}.vtt`} />
|
<track kind="captions" srcLang="en" label={t('common.captions')} src={`/video/${slug}.vtt`} />
|
||||||
</video>
|
</video>
|
||||||
{!isPlaying && (
|
{!isPlaying && (
|
||||||
<div className="pointer-events-none absolute inset-0 flex items-center justify-center bg-navy-950/15 transition-colors">
|
<div className="pointer-events-none absolute inset-0 flex items-center justify-center bg-navy-950/15 transition-colors">
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,8 @@
|
||||||
import {
|
import { useCallback, useMemo, useState, type MutableRefObject, type ReactNode } from 'react';
|
||||||
useCallback,
|
|
||||||
useEffect,
|
|
||||||
useMemo,
|
|
||||||
useRef,
|
|
||||||
useState,
|
|
||||||
type MutableRefObject,
|
|
||||||
type ReactNode,
|
|
||||||
} from 'react';
|
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
import { ts } from '../../i18n/server';
|
import { ts } from '../../i18n/server';
|
||||||
import type {
|
import type {
|
||||||
|
CrimeRecordsResponse,
|
||||||
FeatureFilters,
|
FeatureFilters,
|
||||||
FeatureGroup,
|
FeatureGroup,
|
||||||
FeatureMeta,
|
FeatureMeta,
|
||||||
|
|
@ -39,12 +32,14 @@ import {
|
||||||
} from '../../lib/consts';
|
} from '../../lib/consts';
|
||||||
import { useNearbyStations } from '../../hooks/useNearbyStations';
|
import { useNearbyStations } from '../../hooks/useNearbyStations';
|
||||||
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
|
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
|
||||||
|
import { useRevealOnExpand } from '../../hooks/useRevealOnExpand';
|
||||||
import { DualHistogram, LoadingSkeleton } from './DualHistogram';
|
import { DualHistogram, LoadingSkeleton } from './DualHistogram';
|
||||||
import EnumBarChart from './EnumBarChart';
|
import EnumBarChart from './EnumBarChart';
|
||||||
import StackedBarChart from './StackedBarChart';
|
import StackedBarChart from './StackedBarChart';
|
||||||
import StackedEnumChart from './StackedEnumChart';
|
import StackedEnumChart from './StackedEnumChart';
|
||||||
import PriceHistoryChart from './PriceHistoryChart';
|
import PriceHistoryChart from './PriceHistoryChart';
|
||||||
import CrimeYearChart from './CrimeYearChart';
|
import CrimeYearChart from './CrimeYearChart';
|
||||||
|
import CrimeGroupBody from './CrimeGroupBody';
|
||||||
import NumberLine, { type NumberLinePoint } from './NumberLine';
|
import NumberLine, { type NumberLinePoint } from './NumberLine';
|
||||||
import ExternalSearchLinks from './ExternalSearchLinks';
|
import ExternalSearchLinks from './ExternalSearchLinks';
|
||||||
import { InfoIcon, TransitIcon } from '../ui/icons';
|
import { InfoIcon, TransitIcon } from '../ui/icons';
|
||||||
|
|
@ -72,6 +67,7 @@ interface AreaPaneProps {
|
||||||
shareCode?: string;
|
shareCode?: string;
|
||||||
isGroupExpanded: (name: string) => boolean;
|
isGroupExpanded: (name: string) => boolean;
|
||||||
onToggleGroup: (name: string) => void;
|
onToggleGroup: (name: string) => void;
|
||||||
|
onLoadCrimeRecords?: (offset: number) => Promise<CrimeRecordsResponse>;
|
||||||
scrollTopRef?: MutableRefObject<number>;
|
scrollTopRef?: MutableRefObject<number>;
|
||||||
scrollRestoreKey?: string | null;
|
scrollRestoreKey?: string | null;
|
||||||
scrollSaveDisabled?: boolean;
|
scrollSaveDisabled?: boolean;
|
||||||
|
|
@ -242,6 +238,7 @@ export default function AreaPane({
|
||||||
shareCode,
|
shareCode,
|
||||||
isGroupExpanded,
|
isGroupExpanded,
|
||||||
onToggleGroup,
|
onToggleGroup,
|
||||||
|
onLoadCrimeRecords,
|
||||||
scrollTopRef,
|
scrollTopRef,
|
||||||
scrollRestoreKey,
|
scrollRestoreKey,
|
||||||
scrollSaveDisabled,
|
scrollSaveDisabled,
|
||||||
|
|
@ -297,61 +294,22 @@ export default function AreaPane({
|
||||||
// When the user expands a group, scroll the bottom of the newly opened content
|
// When the user expands a group, scroll the bottom of the newly opened content
|
||||||
// into view so the whole group is revealed without manual scrolling. The group
|
// into view so the whole group is revealed without manual scrolling. The group
|
||||||
// header stays pinned at the top via its sticky positioning.
|
// header stays pinned at the top via its sticky positioning.
|
||||||
const scrollContainerRef = useRef<HTMLDivElement | null>(null);
|
const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
|
||||||
const setScrollNode = useCallback(
|
const setScrollNode = useCallback(
|
||||||
(node: HTMLDivElement | null) => {
|
(node: HTMLDivElement | null) => {
|
||||||
scrollContainerRef.current = node;
|
setContainer(node);
|
||||||
scrollRef(node);
|
scrollRef(node);
|
||||||
},
|
},
|
||||||
[scrollRef]
|
[setContainer, scrollRef]
|
||||||
);
|
);
|
||||||
const groupRefs = useRef<Map<string, HTMLDivElement>>(new Map());
|
|
||||||
// A fresh object each toggle so re-expanding the same group still re-triggers.
|
|
||||||
const [groupToReveal, setGroupToReveal] = useState<{ name: string } | null>(null);
|
|
||||||
const handleToggleGroup = useCallback(
|
const handleToggleGroup = useCallback(
|
||||||
(name: string) => {
|
(name: string) => {
|
||||||
const willExpand = !isGroupExpanded(name);
|
const willExpand = !isGroupExpanded(name);
|
||||||
onToggleGroup(name);
|
onToggleGroup(name);
|
||||||
setGroupToReveal(willExpand ? { name } : null);
|
revealGroupOnToggle(name, willExpand);
|
||||||
},
|
},
|
||||||
[isGroupExpanded, onToggleGroup]
|
[isGroupExpanded, onToggleGroup, revealGroupOnToggle]
|
||||||
);
|
);
|
||||||
useEffect(() => {
|
|
||||||
if (!groupToReveal) return;
|
|
||||||
const el = groupRefs.current.get(groupToReveal.name);
|
|
||||||
const container = scrollContainerRef.current;
|
|
||||||
if (!el || !container) return;
|
|
||||||
|
|
||||||
// Scroll down just enough to bring the bottom of the opened group into view.
|
|
||||||
// For groups taller than the pane this scrolls past the header, but it stays
|
|
||||||
// visible via its sticky positioning. Never scroll up (group already in view).
|
|
||||||
const alignBottom = () => {
|
|
||||||
const delta = el.getBoundingClientRect().bottom - container.getBoundingClientRect().bottom;
|
|
||||||
if (delta <= 1) return;
|
|
||||||
container.scrollTo({ top: container.scrollTop + delta });
|
|
||||||
};
|
|
||||||
|
|
||||||
alignBottom();
|
|
||||||
|
|
||||||
// The group's charts and async data (e.g. nearby stations) can grow its
|
|
||||||
// height after this first pass, so keep the bottom pinned as it settles.
|
|
||||||
// Stop once the user scrolls or after a short grace period so we never fight
|
|
||||||
// a deliberate scroll.
|
|
||||||
let timer = 0;
|
|
||||||
const observer = new ResizeObserver(alignBottom);
|
|
||||||
const stop = () => {
|
|
||||||
observer.disconnect();
|
|
||||||
container.removeEventListener('wheel', stop);
|
|
||||||
container.removeEventListener('touchmove', stop);
|
|
||||||
window.clearTimeout(timer);
|
|
||||||
};
|
|
||||||
observer.observe(el);
|
|
||||||
container.addEventListener('wheel', stop, { passive: true });
|
|
||||||
container.addEventListener('touchmove', stop, { passive: true });
|
|
||||||
timer = window.setTimeout(stop, 1500);
|
|
||||||
|
|
||||||
return stop;
|
|
||||||
}, [groupToReveal]);
|
|
||||||
|
|
||||||
const numericByName = useMemo(() => {
|
const numericByName = useMemo(() => {
|
||||||
if (!stats) return new Map();
|
if (!stats) return new Map();
|
||||||
|
|
@ -363,26 +321,21 @@ export default function AreaPane({
|
||||||
return new Map(stats.enum_features.map((feature) => [feature.name, feature]));
|
return new Map(stats.enum_features.map((feature) => [feature.name, feature]));
|
||||||
}, [stats]);
|
}, [stats]);
|
||||||
|
|
||||||
// Crime-by-year series is keyed in the API by the bare crime type (e.g. "Burglary").
|
// Crime-by-year series, keyed by the bare crime type (e.g. "Burglary").
|
||||||
// We also index by the configured feature name (with " (avg/yr)" suffix) so the
|
const crimeByYearByType = useMemo(() => {
|
||||||
// metric-row renderer can look it up using the feature name it already has.
|
|
||||||
const crimeByYearByFeatureName = useMemo(() => {
|
|
||||||
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_by_year']>[number]>();
|
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_by_year']>[number]>();
|
||||||
for (const entry of stats?.crime_by_year ?? []) {
|
for (const entry of stats?.crime_by_year ?? []) {
|
||||||
map.set(entry.name, entry);
|
map.set(entry.name, entry);
|
||||||
map.set(`${entry.name} (avg/yr)`, entry);
|
|
||||||
}
|
}
|
||||||
return map;
|
return map;
|
||||||
}, [stats]);
|
}, [stats]);
|
||||||
|
|
||||||
// Per-crime-type outcode/sector averages, keyed by both the bare crime type
|
// Per-rate-feature outcode/sector averages, keyed by the FULL rate-feature name
|
||||||
// and the " (avg/yr)" feature name (same convention as crimeByYearByFeatureName)
|
// (e.g. "Burglary (/yr, 7y)") the comparison is computed against.
|
||||||
// so renderers can look them up by the feature name they already hold.
|
|
||||||
const crimeAreaAvgByName = useMemo(() => {
|
const crimeAreaAvgByName = useMemo(() => {
|
||||||
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_area_averages']>[number]>();
|
const map = new Map<string, NonNullable<HexagonStatsResponse['crime_area_averages']>[number]>();
|
||||||
for (const entry of stats?.crime_area_averages ?? []) {
|
for (const entry of stats?.crime_area_averages ?? []) {
|
||||||
map.set(entry.name, entry);
|
map.set(entry.name, entry);
|
||||||
map.set(`${entry.name} (avg/yr)`, entry);
|
|
||||||
}
|
}
|
||||||
return map;
|
return map;
|
||||||
}, [stats]);
|
}, [stats]);
|
||||||
|
|
@ -440,7 +393,11 @@ export default function AreaPane({
|
||||||
<>
|
<>
|
||||||
<div className="relative flex h-full flex-col">
|
<div className="relative flex h-full flex-col">
|
||||||
<IndeterminateProgressBar show={loading && stats != null} />
|
<IndeterminateProgressBar show={loading && stats != null} />
|
||||||
<div ref={setScrollNode} onScroll={onScroll} className="flex-1 overflow-y-auto">
|
<div
|
||||||
|
ref={setScrollNode}
|
||||||
|
onScroll={onScroll}
|
||||||
|
className="flex-1 overflow-y-auto pb-[env(safe-area-inset-bottom)]"
|
||||||
|
>
|
||||||
<div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950">
|
<div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950">
|
||||||
<div className="space-y-3 p-3">
|
<div className="space-y-3 p-3">
|
||||||
<div className="flex items-start justify-between gap-3">
|
<div className="flex items-start justify-between gap-3">
|
||||||
|
|
@ -615,13 +572,7 @@ export default function AreaPane({
|
||||||
);
|
);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div key={group.name} ref={registerGroup(group.name)}>
|
||||||
key={group.name}
|
|
||||||
ref={(el) => {
|
|
||||||
if (el) groupRefs.current.set(group.name, el);
|
|
||||||
else groupRefs.current.delete(group.name);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<CollapsibleGroupHeader
|
<CollapsibleGroupHeader
|
||||||
name={group.name}
|
name={group.name}
|
||||||
expanded={expanded}
|
expanded={expanded}
|
||||||
|
|
@ -631,6 +582,19 @@ export default function AreaPane({
|
||||||
{expanded && (
|
{expanded && (
|
||||||
<div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800">
|
<div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800">
|
||||||
{showNearbyStations && <NearbyStationsCard location={hexagonLocation} />}
|
{showNearbyStations && <NearbyStationsCard location={hexagonLocation} />}
|
||||||
|
{group.name === 'Crime' ? (
|
||||||
|
<CrimeGroupBody
|
||||||
|
stats={stats}
|
||||||
|
numericByName={numericByName}
|
||||||
|
crimeAreaAvgByName={crimeAreaAvgByName}
|
||||||
|
crimeByYearByType={crimeByYearByType}
|
||||||
|
globalFeatureByName={globalFeatureByName}
|
||||||
|
onShowInfo={setInfoFeature}
|
||||||
|
onLoadCrimeRecords={onLoadCrimeRecords}
|
||||||
|
selectionKey={`${isPostcode ? 'postcode' : 'hexagon'}:${hexagonId}`}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
{stackedCharts?.map((chart) => {
|
{stackedCharts?.map((chart) => {
|
||||||
const segments = chart.components
|
const segments = chart.components
|
||||||
.map((name) => ({
|
.map((name) => ({
|
||||||
|
|
@ -651,7 +615,7 @@ export default function AreaPane({
|
||||||
? aggregateStats.mean
|
? aggregateStats.mean
|
||||||
: displaySegments.reduce((sum, s) => sum + s.value, 0);
|
: displaySegments.reduce((sum, s) => sum + s.value, 0);
|
||||||
|
|
||||||
// Use rateFeature (e.g. per-1k) for display if available
|
// Use rateFeature (e.g. a percentage) for display if available
|
||||||
const rateStats = chart.rateFeature
|
const rateStats = chart.rateFeature
|
||||||
? numericByName.get(chart.rateFeature)
|
? numericByName.get(chart.rateFeature)
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
@ -715,7 +679,7 @@ export default function AreaPane({
|
||||||
if (total === 0) return null;
|
if (total === 0) return null;
|
||||||
|
|
||||||
const crimeSeries = chart.feature
|
const crimeSeries = chart.feature
|
||||||
? crimeByYearByFeatureName.get(chart.feature)
|
? crimeByYearByType.get(chart.feature)
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|
@ -800,7 +764,7 @@ export default function AreaPane({
|
||||||
const globalMean = globalHistogram
|
const globalMean = globalHistogram
|
||||||
? calculateHistogramMean(globalHistogram)
|
? calculateHistogramMean(globalHistogram)
|
||||||
: undefined;
|
: undefined;
|
||||||
const crimeSeries = crimeByYearByFeatureName.get(feature.name);
|
const crimeSeries = crimeByYearByType.get(feature.name);
|
||||||
const crimeAreaAvg = crimeAreaAvgByName.get(feature.name);
|
const crimeAreaAvg = crimeAreaAvgByName.get(feature.name);
|
||||||
// National avg is shown for every metric here as a
|
// National avg is shown for every metric here as a
|
||||||
// tooltip; for crime metrics the outcode and sector
|
// tooltip; for crime metrics the outcode and sector
|
||||||
|
|
@ -992,6 +956,8 @@ export default function AreaPane({
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
204
frontend/src/components/map/CrimeGroupBody.tsx
Normal file
204
frontend/src/components/map/CrimeGroupBody.tsx
Normal file
|
|
@ -0,0 +1,204 @@
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
import { ts } from '../../i18n/server';
|
||||||
|
import type {
|
||||||
|
CrimeAreaAverage,
|
||||||
|
CrimeRecordsResponse,
|
||||||
|
CrimeYearStats,
|
||||||
|
FeatureMeta,
|
||||||
|
HexagonStatsResponse,
|
||||||
|
NumericFeatureStats,
|
||||||
|
} from '../../types';
|
||||||
|
import { STACKED_GROUPS, STACKED_SEGMENT_COLORS } from '../../lib/consts';
|
||||||
|
import { formatValue, calculateHistogramMean } from '../../lib/format';
|
||||||
|
import { FeatureLabel } from '../ui/FeatureLabel';
|
||||||
|
import StackedBarChart from './StackedBarChart';
|
||||||
|
import NumberLine, { type NumberLinePoint } from './NumberLine';
|
||||||
|
import CrimeYearChart from './CrimeYearChart';
|
||||||
|
import CrimeRecordsSection from './CrimeRecordsSection';
|
||||||
|
|
||||||
|
type CrimeWindow = '7y' | '2y';
|
||||||
|
|
||||||
|
/** Strip the " (/yr, 7y|2y)" suffix to the bare crime type. */
|
||||||
|
function bareType(featureName: string): string {
|
||||||
|
return featureName.replace(/ \(\/yr, \dy\)$/, '');
|
||||||
|
}
|
||||||
|
/** Re-target a crime-feature name at the chosen averaging window. */
|
||||||
|
function toWindow(featureName: string, window: CrimeWindow): string {
|
||||||
|
return featureName.replace(/(\/yr, )\dy/, `$1${window}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Segment colours keyed by the BARE crime type. The stacked bar then labels each
|
||||||
|
* segment with the clean crime name (e.g. "Burglary") and keeps one stable colour
|
||||||
|
* regardless of whether the 7-year or 2-year window is selected.
|
||||||
|
*/
|
||||||
|
const SEGMENT_COLOR_BY_BARE: Record<string, string> = Object.fromEntries(
|
||||||
|
Object.entries(STACKED_SEGMENT_COLORS).map(([name, color]) => [bareType(name), color])
|
||||||
|
);
|
||||||
|
|
||||||
|
interface CrimeGroupBodyProps {
|
||||||
|
stats: HexagonStatsResponse;
|
||||||
|
numericByName: Map<string, NumericFeatureStats>;
|
||||||
|
/** Keyed by the FULL crime-feature name (e.g. "Burglary (/yr, 7y)"). */
|
||||||
|
crimeAreaAvgByName: Map<string, CrimeAreaAverage>;
|
||||||
|
/** Keyed by the bare crime type (e.g. "Burglary"). */
|
||||||
|
crimeByYearByType: Map<string, CrimeYearStats>;
|
||||||
|
globalFeatureByName: Map<string, FeatureMeta>;
|
||||||
|
onShowInfo: (feature: FeatureMeta) => void;
|
||||||
|
onLoadCrimeRecords?: (offset: number) => Promise<CrimeRecordsResponse>;
|
||||||
|
selectionKey: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function CrimeGroupBody({
|
||||||
|
stats,
|
||||||
|
numericByName,
|
||||||
|
crimeAreaAvgByName,
|
||||||
|
crimeByYearByType,
|
||||||
|
globalFeatureByName,
|
||||||
|
onShowInfo,
|
||||||
|
onLoadCrimeRecords,
|
||||||
|
selectionKey,
|
||||||
|
}: CrimeGroupBodyProps) {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
// One selector drives every value and comparison in this group.
|
||||||
|
const [crimeWindow, setCrimeWindow] = useState<CrimeWindow>('7y');
|
||||||
|
|
||||||
|
const fmtCount = (value?: number) => (value == null ? '—' : formatValue(value));
|
||||||
|
|
||||||
|
const rollupCards = STACKED_GROUPS.Crime ?? [];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div className="flex items-center justify-between gap-2 py-2">
|
||||||
|
<span className="text-[11px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
||||||
|
{t('areaPane.crimeWindowLabel')}
|
||||||
|
</span>
|
||||||
|
<div className="grid grid-cols-2 gap-0.5 rounded-md bg-warm-200 p-0.5 dark:bg-navy-800">
|
||||||
|
{(['7y', '2y'] as const).map((w) => {
|
||||||
|
const active = crimeWindow === w;
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={w}
|
||||||
|
type="button"
|
||||||
|
aria-pressed={active}
|
||||||
|
onClick={() => setCrimeWindow(w)}
|
||||||
|
className={`rounded px-3 py-1 text-xs font-medium ${
|
||||||
|
active
|
||||||
|
? 'bg-white text-teal-700 shadow-sm dark:bg-navy-700 dark:text-teal-300'
|
||||||
|
: 'text-warm-600 hover:text-warm-900 dark:text-warm-400 dark:hover:text-warm-100'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{t(w === '7y' ? 'areaPane.crimeWindow7y' : 'areaPane.crimeWindow2y')}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{rollupCards.map((card) => {
|
||||||
|
if (!card.feature) return null;
|
||||||
|
const bare = bareType(card.feature);
|
||||||
|
const windowFeature = toWindow(card.feature, crimeWindow);
|
||||||
|
const rate = numericByName.get(windowFeature)?.mean;
|
||||||
|
|
||||||
|
const segments = card.components
|
||||||
|
.map((name) => ({
|
||||||
|
name: bareType(name),
|
||||||
|
value: numericByName.get(toWindow(name, crimeWindow))?.mean ?? 0,
|
||||||
|
}))
|
||||||
|
.filter((s) => s.value > 0);
|
||||||
|
const total = segments.reduce((sum, s) => sum + s.value, 0);
|
||||||
|
if (rate == null && total === 0) return null;
|
||||||
|
|
||||||
|
const displayValue = rate ?? total;
|
||||||
|
// Info popup / icon stay anchored to the 7-year feature so the metric
|
||||||
|
// definition is stable across windows; only the numbers change.
|
||||||
|
const featureMeta = globalFeatureByName.get(card.feature);
|
||||||
|
const globalMean = featureMeta?.histogram
|
||||||
|
? calculateHistogramMean(featureMeta.histogram)
|
||||||
|
: undefined;
|
||||||
|
const crimeAreaAvg = crimeAreaAvgByName.get(windowFeature);
|
||||||
|
const nationalAvg = crimeAreaAvg?.national ?? globalMean;
|
||||||
|
const cardTitle = t('areaPane.crimeCardTitle', { name: ts(card.label) });
|
||||||
|
|
||||||
|
const numberLinePoints: NumberLinePoint[] = crimeAreaAvg
|
||||||
|
? (
|
||||||
|
[
|
||||||
|
{ kind: 'area', label: t('areaPane.thisArea'), value: displayValue },
|
||||||
|
nationalAvg != null
|
||||||
|
? { kind: 'national', label: t('areaPane.national'), value: nationalAvg }
|
||||||
|
: null,
|
||||||
|
crimeAreaAvg.outcode != null
|
||||||
|
? {
|
||||||
|
kind: 'outcode',
|
||||||
|
label: stats.crime_outcode ?? t('areaPane.outcodeAvg'),
|
||||||
|
value: crimeAreaAvg.outcode,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
crimeAreaAvg.sector != null
|
||||||
|
? {
|
||||||
|
kind: 'sector',
|
||||||
|
label: stats.crime_sector ?? t('areaPane.sectorAvg'),
|
||||||
|
value: crimeAreaAvg.sector,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
] as (NumberLinePoint | null)[]
|
||||||
|
).filter((p): p is NumberLinePoint => p !== null)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const series = crimeByYearByType.get(bare);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div key={card.label} className="rounded bg-warm-50 p-2 dark:bg-warm-800">
|
||||||
|
<div className="mb-1.5 flex items-baseline justify-between gap-2">
|
||||||
|
{featureMeta ? (
|
||||||
|
<FeatureLabel
|
||||||
|
feature={{ ...featureMeta, name: ts(card.label) }}
|
||||||
|
label={cardTitle}
|
||||||
|
onShowInfo={onShowInfo}
|
||||||
|
className="mr-2"
|
||||||
|
wrap
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<span className="mr-2 min-w-0 break-words text-xs leading-snug text-warm-700 dark:text-warm-300">
|
||||||
|
{cardTitle}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
<div className="shrink-0 text-right">
|
||||||
|
<span className="whitespace-nowrap text-xs font-semibold text-teal-700 dark:text-teal-400">
|
||||||
|
{fmtCount(displayValue)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{total > 0 && (
|
||||||
|
<StackedBarChart segments={segments} total={total} colorMap={SEGMENT_COLOR_BY_BARE} />
|
||||||
|
)}
|
||||||
|
{numberLinePoints.length >= 2 && (
|
||||||
|
<div className="mt-2">
|
||||||
|
<NumberLine points={numberLinePoints} format={formatValue} />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{series && series.points.length > 1 && (
|
||||||
|
<div className="mt-2">
|
||||||
|
<CrimeYearChart
|
||||||
|
points={series.points}
|
||||||
|
latestAvailableYear={stats.crime_latest_year}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
|
||||||
|
{onLoadCrimeRecords && (
|
||||||
|
<CrimeRecordsSection
|
||||||
|
selectionKey={selectionKey}
|
||||||
|
total={stats.crime_total_records}
|
||||||
|
onLoad={onLoadCrimeRecords}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
147
frontend/src/components/map/CrimeRecordsSection.tsx
Normal file
147
frontend/src/components/map/CrimeRecordsSection.tsx
Normal file
|
|
@ -0,0 +1,147 @@
|
||||||
|
import { useEffect, useRef, useState } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
import { ts } from '../../i18n/server';
|
||||||
|
import type { CrimeIncident, CrimeRecordsResponse } from '../../types';
|
||||||
|
import { isAbortError, logNonAbortError } from '../../lib/api';
|
||||||
|
|
||||||
|
function formatMonth(month: string, locale: string): string {
|
||||||
|
const [year, m] = month.split('-').map(Number);
|
||||||
|
if (!year || !m) return month;
|
||||||
|
return new Date(Date.UTC(year, m - 1, 1)).toLocaleDateString(locale, {
|
||||||
|
year: 'numeric',
|
||||||
|
month: 'short',
|
||||||
|
timeZone: 'UTC',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CrimeRecordsSectionProps {
|
||||||
|
/** Identity of the current selection; changing it resets the list. */
|
||||||
|
selectionKey: string | null;
|
||||||
|
/** Total records for the selection (from stats); the section hides when 0. */
|
||||||
|
total?: number;
|
||||||
|
onLoad: (offset: number) => Promise<CrimeRecordsResponse>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function CrimeRecordsSection({
|
||||||
|
selectionKey,
|
||||||
|
total,
|
||||||
|
onLoad,
|
||||||
|
}: CrimeRecordsSectionProps) {
|
||||||
|
const { t, i18n } = useTranslation();
|
||||||
|
const [open, setOpen] = useState(false);
|
||||||
|
const [records, setRecords] = useState<CrimeIncident[]>([]);
|
||||||
|
const [loadedTotal, setLoadedTotal] = useState(0);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState(false);
|
||||||
|
// Bumped whenever the selection changes; in-flight responses for a previous
|
||||||
|
// selection are ignored when they resolve.
|
||||||
|
const selectionIdRef = useRef(0);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
selectionIdRef.current += 1;
|
||||||
|
setOpen(false);
|
||||||
|
setRecords([]);
|
||||||
|
setLoadedTotal(0);
|
||||||
|
setLoading(false);
|
||||||
|
setError(false);
|
||||||
|
}, [selectionKey]);
|
||||||
|
|
||||||
|
if (!total || total <= 0) return null;
|
||||||
|
|
||||||
|
const fetchPage = async (offset: number) => {
|
||||||
|
const myId = selectionIdRef.current;
|
||||||
|
setLoading(true);
|
||||||
|
setError(false);
|
||||||
|
try {
|
||||||
|
const data = await onLoad(offset);
|
||||||
|
if (myId !== selectionIdRef.current) return;
|
||||||
|
setRecords((prev) => (offset === 0 ? data.records : [...prev, ...data.records]));
|
||||||
|
setLoadedTotal(data.total);
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) return;
|
||||||
|
logNonAbortError('Failed to fetch crime records', err);
|
||||||
|
if (myId === selectionIdRef.current) setError(true);
|
||||||
|
} finally {
|
||||||
|
if (myId === selectionIdRef.current) setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleToggle = () => {
|
||||||
|
const next = !open;
|
||||||
|
setOpen(next);
|
||||||
|
if (next && records.length === 0 && !loading) fetchPage(0);
|
||||||
|
};
|
||||||
|
|
||||||
|
const shownTotal = loadedTotal || total;
|
||||||
|
const canLoadMore = records.length < loadedTotal;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="py-1.5">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={handleToggle}
|
||||||
|
className="flex w-full items-center justify-between gap-2 rounded px-1 py-1 text-left hover:bg-warm-100 dark:hover:bg-navy-800"
|
||||||
|
aria-expanded={open}
|
||||||
|
>
|
||||||
|
<span className="text-[13px] font-medium text-warm-900 dark:text-warm-100">
|
||||||
|
{open ? t('areaPane.crimeRecordsHide') : t('areaPane.crimeRecordsToggle')}
|
||||||
|
</span>
|
||||||
|
<span className="shrink-0 text-xs font-semibold tabular-nums text-warm-500 dark:text-warm-400">
|
||||||
|
{t('areaPane.crimeRecordsCount', { count: shownTotal })}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{open && (
|
||||||
|
<div className="mt-1.5">
|
||||||
|
{error ? (
|
||||||
|
<div className="py-2 text-sm text-rose-600 dark:text-rose-400">
|
||||||
|
{t('areaPane.crimeRecordsError')}
|
||||||
|
</div>
|
||||||
|
) : records.length === 0 && loading ? (
|
||||||
|
<div className="flex items-center gap-2 py-2 text-sm text-warm-500 dark:text-warm-400">
|
||||||
|
<span className="h-3 w-3 animate-spin rounded-full border-2 border-teal-600 border-t-transparent dark:border-teal-400 dark:border-t-transparent" />
|
||||||
|
{t('areaPane.crimeRecordsLoading')}
|
||||||
|
</div>
|
||||||
|
) : records.length === 0 ? (
|
||||||
|
<div className="py-2 text-sm text-warm-500 dark:text-warm-400">
|
||||||
|
{t('areaPane.crimeRecordsEmpty')}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<ol className="divide-y divide-warm-100 dark:divide-navy-800">
|
||||||
|
{records.map((record, index) => (
|
||||||
|
<li key={`${record.month}-${record.lat}-${record.lon}-${index}`} className="py-1.5">
|
||||||
|
<div className="flex items-baseline justify-between gap-2">
|
||||||
|
<span className="min-w-0 break-words text-[13px] font-medium text-warm-900 dark:text-warm-100">
|
||||||
|
{ts(record.type)}
|
||||||
|
</span>
|
||||||
|
<span className="shrink-0 text-xs tabular-nums text-warm-500 dark:text-warm-400">
|
||||||
|
{formatMonth(record.month, i18n.language)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="mt-0.5 text-xs text-warm-500 dark:text-warm-400">
|
||||||
|
{record.location ?? t('areaPane.crimeNoLocation')}
|
||||||
|
</div>
|
||||||
|
<div className="text-xs text-warm-400 dark:text-warm-500">
|
||||||
|
{record.outcome ?? t('areaPane.crimeOutcomeUnknown')}
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ol>
|
||||||
|
{canLoadMore && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => !loading && fetchPage(records.length)}
|
||||||
|
disabled={loading}
|
||||||
|
className="mt-2 w-full rounded border border-warm-200 px-2 py-1.5 text-xs font-medium text-warm-700 hover:bg-warm-100 disabled:opacity-50 dark:border-navy-700 dark:text-warm-200 dark:hover:bg-navy-800"
|
||||||
|
>
|
||||||
|
{loading ? t('areaPane.crimeRecordsLoading') : t('areaPane.crimeRecordsLoadMore')}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -82,7 +82,12 @@ export default function CrimeYearChart({ points, latestAvailableYear }: CrimeYea
|
||||||
r={1.6}
|
r={1.6}
|
||||||
className="fill-rose-700 dark:fill-rose-300"
|
className="fill-rose-700 dark:fill-rose-300"
|
||||||
>
|
>
|
||||||
<title>{`${p.year}: ${p.count.toFixed(1)}/yr`}</title>
|
<title>
|
||||||
|
{t('areaPane.crimeYearPointTooltip', {
|
||||||
|
year: p.year,
|
||||||
|
rate: p.count.toFixed(1),
|
||||||
|
})}
|
||||||
|
</title>
|
||||||
</circle>
|
</circle>
|
||||||
))}
|
))}
|
||||||
<text
|
<text
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ export default function ExternalSearchLinks({
|
||||||
const radiusMiles = location.isPostcode
|
const radiusMiles = location.isPostcode
|
||||||
? POSTCODE_RADIUS_MILES
|
? POSTCODE_RADIUS_MILES
|
||||||
: (H3_RADIUS_MILES[location.resolution] ?? 1);
|
: (H3_RADIUS_MILES[location.resolution] ?? 1);
|
||||||
const label = `${radiusMiles}mi radius`;
|
const label = t('externalSearch.radiusMi', { count: radiusMiles });
|
||||||
|
|
||||||
if (!urls) return null;
|
if (!urls) return null;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,10 @@ interface FeatureBrowserProps {
|
||||||
onClearOpenInfoFeature?: () => void;
|
onClearOpenInfoFeature?: () => void;
|
||||||
travelTimeEntries: TravelTimeEntry[];
|
travelTimeEntries: TravelTimeEntry[];
|
||||||
onAddTravelTimeEntry: (mode: TransportMode) => void;
|
onAddTravelTimeEntry: (mode: TransportMode) => void;
|
||||||
|
/** Registers a group wrapper so an expanded group can be scrolled into view. */
|
||||||
|
registerGroup?: (name: string) => (node: HTMLDivElement | null) => void;
|
||||||
|
/** Notifies the reveal-on-expand mechanism when a group is toggled. */
|
||||||
|
onGroupToggle?: (name: string, willExpand: boolean) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function FeatureBrowser({
|
export default function FeatureBrowser({
|
||||||
|
|
@ -46,6 +50,8 @@ export default function FeatureBrowser({
|
||||||
onClearOpenInfoFeature,
|
onClearOpenInfoFeature,
|
||||||
travelTimeEntries: _travelTimeEntries,
|
travelTimeEntries: _travelTimeEntries,
|
||||||
onAddTravelTimeEntry,
|
onAddTravelTimeEntry,
|
||||||
|
registerGroup,
|
||||||
|
onGroupToggle,
|
||||||
}: FeatureBrowserProps) {
|
}: FeatureBrowserProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const modes = useTranslatedModes();
|
const modes = useTranslatedModes();
|
||||||
|
|
@ -115,11 +121,15 @@ export default function FeatureBrowser({
|
||||||
{mergedGrouped.map((group) => {
|
{mergedGrouped.map((group) => {
|
||||||
const isExpanded = isSearching || isGroupExpanded(group.name);
|
const isExpanded = isSearching || isGroupExpanded(group.name);
|
||||||
return (
|
return (
|
||||||
<div key={group.name} className="shrink-0">
|
<div key={group.name} className="shrink-0" ref={registerGroup?.(group.name)}>
|
||||||
<CollapsibleGroupHeader
|
<CollapsibleGroupHeader
|
||||||
name={group.name}
|
name={group.name}
|
||||||
expanded={isExpanded}
|
expanded={isExpanded}
|
||||||
onToggle={() => toggleGroup(group.name)}
|
onToggle={() => {
|
||||||
|
const willExpand = !isExpanded;
|
||||||
|
toggleGroup(group.name);
|
||||||
|
onGroupToggle?.(group.name, willExpand);
|
||||||
|
}}
|
||||||
className="px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 sticky top-0 z-30 hover:bg-warm-200 dark:hover:bg-warm-800"
|
className="px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 sticky top-0 z-30 hover:bg-warm-200 dark:hover:bg-warm-800"
|
||||||
>
|
>
|
||||||
<span className="text-xs font-medium text-warm-400 dark:text-warm-500">
|
<span className="text-xs font-medium text-warm-400 dark:text-warm-500">
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,16 @@ import {
|
||||||
isSpecificCrimeFeatureName,
|
isSpecificCrimeFeatureName,
|
||||||
isSpecificCrimeFilterName,
|
isSpecificCrimeFilterName,
|
||||||
} from '../../lib/crime-filter';
|
} from '../../lib/crime-filter';
|
||||||
|
import {
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES,
|
||||||
|
getCrimeSeverityFeatureName,
|
||||||
|
getCrimeSeverityFilterMeta,
|
||||||
|
getCrimeSeverityFilterName,
|
||||||
|
getDefaultCrimeSeverityFeatureName,
|
||||||
|
isCrimeSeverityFeatureName,
|
||||||
|
isCrimeSeverityFilterName,
|
||||||
|
type CrimeSeverityFilterName,
|
||||||
|
} from '../../lib/crime-severity-filter';
|
||||||
import {
|
import {
|
||||||
ELECTION_VOTE_SHARE_FILTER_NAME,
|
ELECTION_VOTE_SHARE_FILTER_NAME,
|
||||||
getDefaultElectionVoteShareFeatureName,
|
getDefaultElectionVoteShareFeatureName,
|
||||||
|
|
@ -37,7 +47,22 @@ import {
|
||||||
isEthnicityFeatureName,
|
isEthnicityFeatureName,
|
||||||
isEthnicityFilterName,
|
isEthnicityFilterName,
|
||||||
} from '../../lib/ethnicity-filter';
|
} from '../../lib/ethnicity-filter';
|
||||||
import { isQualificationFeatureName } from '../../lib/qualification-filter';
|
import {
|
||||||
|
QUALIFICATIONS_FILTER_NAME,
|
||||||
|
getDefaultQualificationFeatureName,
|
||||||
|
getQualificationFeatureName,
|
||||||
|
getQualificationFilterMeta,
|
||||||
|
isQualificationFeatureName,
|
||||||
|
isQualificationFilterName,
|
||||||
|
} from '../../lib/qualification-filter';
|
||||||
|
import {
|
||||||
|
TENURE_FILTER_NAME,
|
||||||
|
getDefaultTenureFeatureName,
|
||||||
|
getTenureFeatureName,
|
||||||
|
getTenureFilterMeta,
|
||||||
|
isTenureFeatureName,
|
||||||
|
isTenureFilterName,
|
||||||
|
} from '../../lib/tenure-filter';
|
||||||
import {
|
import {
|
||||||
SCHOOL_FILTER_NAME,
|
SCHOOL_FILTER_NAME,
|
||||||
getDefaultSchoolFeatureName,
|
getDefaultSchoolFeatureName,
|
||||||
|
|
@ -182,6 +207,33 @@ export default memo(function Filters({
|
||||||
[features]
|
[features]
|
||||||
);
|
);
|
||||||
const ethnicityMeta = useMemo(() => getEthnicityFilterMeta(features), [features]);
|
const ethnicityMeta = useMemo(() => getEthnicityFilterMeta(features), [features]);
|
||||||
|
const defaultQualificationFeatureName = useMemo(
|
||||||
|
() => getDefaultQualificationFeatureName(features),
|
||||||
|
[features]
|
||||||
|
);
|
||||||
|
const qualificationMeta = useMemo(() => getQualificationFilterMeta(features), [features]);
|
||||||
|
const defaultTenureFeatureName = useMemo(() => getDefaultTenureFeatureName(features), [features]);
|
||||||
|
const tenureMeta = useMemo(() => getTenureFilterMeta(features), [features]);
|
||||||
|
const crimeSeverityMetas = useMemo(
|
||||||
|
() =>
|
||||||
|
Object.fromEntries(
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES.map((filterName) => [
|
||||||
|
filterName,
|
||||||
|
getCrimeSeverityFilterMeta(features, filterName),
|
||||||
|
])
|
||||||
|
) as Record<CrimeSeverityFilterName, FeatureMeta>,
|
||||||
|
[features]
|
||||||
|
);
|
||||||
|
const defaultCrimeSeverityFeatureNames = useMemo(
|
||||||
|
() =>
|
||||||
|
Object.fromEntries(
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES.map((filterName) => [
|
||||||
|
filterName,
|
||||||
|
getDefaultCrimeSeverityFeatureName(features, filterName),
|
||||||
|
])
|
||||||
|
) as Record<CrimeSeverityFilterName, string | null>,
|
||||||
|
[features]
|
||||||
|
);
|
||||||
const defaultPoiDistanceFeatureName = useMemo(
|
const defaultPoiDistanceFeatureName = useMemo(
|
||||||
() => getDefaultPoiDistanceFeatureName(features),
|
() => getDefaultPoiDistanceFeatureName(features),
|
||||||
[features]
|
[features]
|
||||||
|
|
@ -256,6 +308,18 @@ export default memo(function Filters({
|
||||||
return { ...(backendFeature ?? specificCrimeMeta), name, group: 'Crime' };
|
return { ...(backendFeature ?? specificCrimeMeta), name, group: 'Crime' };
|
||||||
});
|
});
|
||||||
}, [filters, features, specificCrimeMeta]);
|
}, [filters, features, specificCrimeMeta]);
|
||||||
|
const crimeSeverityFilterItems = useMemo(() => {
|
||||||
|
return Object.keys(filters)
|
||||||
|
.filter(isCrimeSeverityFilterName)
|
||||||
|
.map((name) => {
|
||||||
|
const backendName = getCrimeSeverityFeatureName(name);
|
||||||
|
const filterName = getCrimeSeverityFilterName(name) ?? CRIME_SEVERITY_FILTER_NAMES[0];
|
||||||
|
const backendFeature = backendName
|
||||||
|
? features.find((feature) => feature.name === backendName)
|
||||||
|
: undefined;
|
||||||
|
return { ...(backendFeature ?? crimeSeverityMetas[filterName]), name, group: 'Crime' };
|
||||||
|
});
|
||||||
|
}, [filters, features, crimeSeverityMetas]);
|
||||||
const electionVoteShareFilterItems = useMemo(() => {
|
const electionVoteShareFilterItems = useMemo(() => {
|
||||||
return Object.keys(filters)
|
return Object.keys(filters)
|
||||||
.filter(isElectionVoteShareFilterName)
|
.filter(isElectionVoteShareFilterName)
|
||||||
|
|
@ -278,6 +342,28 @@ export default memo(function Filters({
|
||||||
return { ...(backendFeature ?? ethnicityMeta), name, group: 'Neighbours' };
|
return { ...(backendFeature ?? ethnicityMeta), name, group: 'Neighbours' };
|
||||||
});
|
});
|
||||||
}, [filters, features, ethnicityMeta]);
|
}, [filters, features, ethnicityMeta]);
|
||||||
|
const qualificationFilterItems = useMemo(() => {
|
||||||
|
return Object.keys(filters)
|
||||||
|
.filter(isQualificationFilterName)
|
||||||
|
.map((name) => {
|
||||||
|
const backendName = getQualificationFeatureName(name);
|
||||||
|
const backendFeature = backendName
|
||||||
|
? features.find((feature) => feature.name === backendName)
|
||||||
|
: undefined;
|
||||||
|
return { ...(backendFeature ?? qualificationMeta), name, group: 'Neighbours' };
|
||||||
|
});
|
||||||
|
}, [filters, features, qualificationMeta]);
|
||||||
|
const tenureFilterItems = useMemo(() => {
|
||||||
|
return Object.keys(filters)
|
||||||
|
.filter(isTenureFilterName)
|
||||||
|
.map((name) => {
|
||||||
|
const backendName = getTenureFeatureName(name);
|
||||||
|
const backendFeature = backendName
|
||||||
|
? features.find((feature) => feature.name === backendName)
|
||||||
|
: undefined;
|
||||||
|
return { ...(backendFeature ?? tenureMeta), name, group: 'Neighbours' };
|
||||||
|
});
|
||||||
|
}, [filters, features, tenureMeta]);
|
||||||
const poiDistanceFilterItems = useMemo(() => {
|
const poiDistanceFilterItems = useMemo(() => {
|
||||||
return Object.keys(filters)
|
return Object.keys(filters)
|
||||||
.filter(isPoiDistanceFilterName)
|
.filter(isPoiDistanceFilterName)
|
||||||
|
|
@ -300,6 +386,8 @@ export default memo(function Filters({
|
||||||
let insertedSpecificCrimeFilter = false;
|
let insertedSpecificCrimeFilter = false;
|
||||||
let insertedElectionVoteShareFilter = false;
|
let insertedElectionVoteShareFilter = false;
|
||||||
let insertedEthnicityFilter = false;
|
let insertedEthnicityFilter = false;
|
||||||
|
let insertedQualificationFilter = false;
|
||||||
|
let insertedTenureFilter = false;
|
||||||
const insertedPoiFilters = new Set<PoiFilterName>();
|
const insertedPoiFilters = new Set<PoiFilterName>();
|
||||||
const maybeInsertPoiFilter = (filterName: PoiFilterName | null) => {
|
const maybeInsertPoiFilter = (filterName: PoiFilterName | null) => {
|
||||||
if (
|
if (
|
||||||
|
|
@ -311,6 +399,17 @@ export default memo(function Filters({
|
||||||
insertedPoiFilters.add(filterName);
|
insertedPoiFilters.add(filterName);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
const insertedCrimeSeverityFilters = new Set<CrimeSeverityFilterName>();
|
||||||
|
const maybeInsertCrimeSeverityFilter = (filterName: CrimeSeverityFilterName | null) => {
|
||||||
|
if (
|
||||||
|
filterName &&
|
||||||
|
defaultCrimeSeverityFeatureNames[filterName] &&
|
||||||
|
!insertedCrimeSeverityFilters.has(filterName)
|
||||||
|
) {
|
||||||
|
result.push(crimeSeverityMetas[filterName]);
|
||||||
|
insertedCrimeSeverityFilters.add(filterName);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
for (const feature of features) {
|
for (const feature of features) {
|
||||||
if (feature.group === 'Transport') {
|
if (feature.group === 'Transport') {
|
||||||
|
|
@ -330,6 +429,12 @@ export default memo(function Filters({
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// "Serious crime" and "Minor crime" each fold their 7y/2y windows into one
|
||||||
|
// card with a period toggle (no variant dropdown — a single feature each).
|
||||||
|
if (isCrimeSeverityFeatureName(feature.name)) {
|
||||||
|
maybeInsertCrimeSeverityFilter(getCrimeSeverityFilterName(feature.name));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (isElectionVoteShareFeatureName(feature.name)) {
|
if (isElectionVoteShareFeatureName(feature.name)) {
|
||||||
if (defaultElectionVoteShareFeatureName && !insertedElectionVoteShareFilter) {
|
if (defaultElectionVoteShareFeatureName && !insertedElectionVoteShareFilter) {
|
||||||
result.push(electionVoteShareMeta);
|
result.push(electionVoteShareMeta);
|
||||||
|
|
@ -344,14 +449,28 @@ export default memo(function Filters({
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// The seven qualification bands fold into one "Qualifications" filter
|
||||||
|
// whose dropdown picks a band, rather than seven separate sliders.
|
||||||
|
if (isQualificationFeatureName(feature.name)) {
|
||||||
|
if (defaultQualificationFeatureName && !insertedQualificationFilter) {
|
||||||
|
result.push(qualificationMeta);
|
||||||
|
insertedQualificationFilter = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The three tenure categories fold into one "Tenure" filter
|
||||||
|
// whose dropdown picks a category, rather than three separate sliders.
|
||||||
|
if (isTenureFeatureName(feature.name)) {
|
||||||
|
if (defaultTenureFeatureName && !insertedTenureFilter) {
|
||||||
|
result.push(tenureMeta);
|
||||||
|
insertedTenureFilter = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (isPoiFilterFeatureName(feature.name)) {
|
if (isPoiFilterFeatureName(feature.name)) {
|
||||||
maybeInsertPoiFilter(getPoiFilterName(feature.name));
|
maybeInsertPoiFilter(getPoiFilterName(feature.name));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Qualification breakdown is display-only (shown as the stacked
|
|
||||||
// "Qualifications" composition in the area pane), so keep its seven
|
|
||||||
// component features out of the filter browser.
|
|
||||||
if (isQualificationFeatureName(feature.name)) continue;
|
|
||||||
if (!enabledFeatures.has(feature.name)) result.push(feature);
|
if (!enabledFeatures.has(feature.name)) result.push(feature);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -363,10 +482,16 @@ export default memo(function Filters({
|
||||||
schoolMeta,
|
schoolMeta,
|
||||||
defaultSpecificCrimeFeatureName,
|
defaultSpecificCrimeFeatureName,
|
||||||
specificCrimeMeta,
|
specificCrimeMeta,
|
||||||
|
defaultCrimeSeverityFeatureNames,
|
||||||
|
crimeSeverityMetas,
|
||||||
defaultElectionVoteShareFeatureName,
|
defaultElectionVoteShareFeatureName,
|
||||||
electionVoteShareMeta,
|
electionVoteShareMeta,
|
||||||
defaultEthnicityFeatureName,
|
defaultEthnicityFeatureName,
|
||||||
ethnicityMeta,
|
ethnicityMeta,
|
||||||
|
defaultQualificationFeatureName,
|
||||||
|
qualificationMeta,
|
||||||
|
defaultTenureFeatureName,
|
||||||
|
tenureMeta,
|
||||||
defaultPoiFilterFeatureNames,
|
defaultPoiFilterFeatureNames,
|
||||||
poiFilterMetas,
|
poiFilterMetas,
|
||||||
]);
|
]);
|
||||||
|
|
@ -376,6 +501,8 @@ export default memo(function Filters({
|
||||||
let insertedSpecificCrimeFilters = false;
|
let insertedSpecificCrimeFilters = false;
|
||||||
let insertedElectionVoteShareFilters = false;
|
let insertedElectionVoteShareFilters = false;
|
||||||
let insertedEthnicityFilters = false;
|
let insertedEthnicityFilters = false;
|
||||||
|
let insertedQualificationFilters = false;
|
||||||
|
let insertedTenureFilters = false;
|
||||||
const insertedPoiFilters = new Set<PoiFilterName>();
|
const insertedPoiFilters = new Set<PoiFilterName>();
|
||||||
const insertPoiFilterItems = (filterName: PoiFilterName | null) => {
|
const insertPoiFilterItems = (filterName: PoiFilterName | null) => {
|
||||||
if (!filterName || insertedPoiFilters.has(filterName)) return;
|
if (!filterName || insertedPoiFilters.has(filterName)) return;
|
||||||
|
|
@ -384,6 +511,16 @@ export default memo(function Filters({
|
||||||
);
|
);
|
||||||
insertedPoiFilters.add(filterName);
|
insertedPoiFilters.add(filterName);
|
||||||
};
|
};
|
||||||
|
const insertedCrimeSeverityFilters = new Set<CrimeSeverityFilterName>();
|
||||||
|
const insertCrimeSeverityFilterItems = (filterName: CrimeSeverityFilterName | null) => {
|
||||||
|
if (!filterName || insertedCrimeSeverityFilters.has(filterName)) return;
|
||||||
|
result.push(
|
||||||
|
...crimeSeverityFilterItems.filter(
|
||||||
|
(item) => getCrimeSeverityFilterName(item.name) === filterName
|
||||||
|
)
|
||||||
|
);
|
||||||
|
insertedCrimeSeverityFilters.add(filterName);
|
||||||
|
};
|
||||||
|
|
||||||
for (const feature of features) {
|
for (const feature of features) {
|
||||||
if (feature.group === 'Transport') {
|
if (feature.group === 'Transport') {
|
||||||
|
|
@ -403,6 +540,10 @@ export default memo(function Filters({
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (isCrimeSeverityFeatureName(feature.name)) {
|
||||||
|
insertCrimeSeverityFilterItems(getCrimeSeverityFilterName(feature.name));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (isElectionVoteShareFeatureName(feature.name)) {
|
if (isElectionVoteShareFeatureName(feature.name)) {
|
||||||
if (!insertedElectionVoteShareFilters) {
|
if (!insertedElectionVoteShareFilters) {
|
||||||
result.push(...electionVoteShareFilterItems);
|
result.push(...electionVoteShareFilterItems);
|
||||||
|
|
@ -417,6 +558,20 @@ export default memo(function Filters({
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (isQualificationFeatureName(feature.name)) {
|
||||||
|
if (!insertedQualificationFilters) {
|
||||||
|
result.push(...qualificationFilterItems);
|
||||||
|
insertedQualificationFilters = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (isTenureFeatureName(feature.name)) {
|
||||||
|
if (!insertedTenureFilters) {
|
||||||
|
result.push(...tenureFilterItems);
|
||||||
|
insertedTenureFilters = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (isPoiFilterFeatureName(feature.name)) {
|
if (isPoiFilterFeatureName(feature.name)) {
|
||||||
insertPoiFilterItems(getPoiFilterName(feature.name));
|
insertPoiFilterItems(getPoiFilterName(feature.name));
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -430,8 +585,11 @@ export default memo(function Filters({
|
||||||
enabledFeatures,
|
enabledFeatures,
|
||||||
schoolFilterItems,
|
schoolFilterItems,
|
||||||
specificCrimeFilterItems,
|
specificCrimeFilterItems,
|
||||||
|
crimeSeverityFilterItems,
|
||||||
electionVoteShareFilterItems,
|
electionVoteShareFilterItems,
|
||||||
ethnicityFilterItems,
|
ethnicityFilterItems,
|
||||||
|
qualificationFilterItems,
|
||||||
|
tenureFilterItems,
|
||||||
poiDistanceFilterItems,
|
poiDistanceFilterItems,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -460,10 +618,15 @@ export default memo(function Filters({
|
||||||
(name: string): string | null => {
|
(name: string): string | null => {
|
||||||
if (name === SCHOOL_FILTER_NAME) return schoolMeta.group ?? 'Schools';
|
if (name === SCHOOL_FILTER_NAME) return schoolMeta.group ?? 'Schools';
|
||||||
if (name === SPECIFIC_CRIMES_FILTER_NAME) return specificCrimeMeta.group ?? 'Crime';
|
if (name === SPECIFIC_CRIMES_FILTER_NAME) return specificCrimeMeta.group ?? 'Crime';
|
||||||
|
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
|
||||||
|
return crimeSeverityMetas[name as CrimeSeverityFilterName].group ?? 'Crime';
|
||||||
|
}
|
||||||
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
|
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
|
||||||
return electionVoteShareMeta.group ?? 'Neighbours';
|
return electionVoteShareMeta.group ?? 'Neighbours';
|
||||||
}
|
}
|
||||||
if (name === ETHNICITIES_FILTER_NAME) return ethnicityMeta.group ?? 'Neighbours';
|
if (name === ETHNICITIES_FILTER_NAME) return ethnicityMeta.group ?? 'Neighbours';
|
||||||
|
if (name === QUALIFICATIONS_FILTER_NAME) return qualificationMeta.group ?? 'Neighbours';
|
||||||
|
if (name === TENURE_FILTER_NAME) return tenureMeta.group ?? 'Neighbours';
|
||||||
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
||||||
return poiFilterMetas[name as PoiFilterName].group ?? null;
|
return poiFilterMetas[name as PoiFilterName].group ?? null;
|
||||||
}
|
}
|
||||||
|
|
@ -472,8 +635,11 @@ export default memo(function Filters({
|
||||||
[
|
[
|
||||||
electionVoteShareMeta.group,
|
electionVoteShareMeta.group,
|
||||||
ethnicityMeta.group,
|
ethnicityMeta.group,
|
||||||
|
qualificationMeta.group,
|
||||||
|
tenureMeta.group,
|
||||||
features,
|
features,
|
||||||
poiFilterMetas,
|
poiFilterMetas,
|
||||||
|
crimeSeverityMetas,
|
||||||
schoolMeta.group,
|
schoolMeta.group,
|
||||||
specificCrimeMeta.group,
|
specificCrimeMeta.group,
|
||||||
]
|
]
|
||||||
|
|
@ -514,6 +680,28 @@ export default memo(function Filters({
|
||||||
onAddFilter(ETHNICITIES_FILTER_NAME);
|
onAddFilter(ETHNICITIES_FILTER_NAME);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (name === QUALIFICATIONS_FILTER_NAME) {
|
||||||
|
if (!defaultQualificationFeatureName) return;
|
||||||
|
queueActiveFilterScroll(
|
||||||
|
QUALIFICATIONS_FILTER_NAME,
|
||||||
|
getAddFilterGroupName(QUALIFICATIONS_FILTER_NAME)
|
||||||
|
);
|
||||||
|
onAddFilter(QUALIFICATIONS_FILTER_NAME);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (name === TENURE_FILTER_NAME) {
|
||||||
|
if (!defaultTenureFeatureName) return;
|
||||||
|
queueActiveFilterScroll(TENURE_FILTER_NAME, getAddFilterGroupName(TENURE_FILTER_NAME));
|
||||||
|
onAddFilter(TENURE_FILTER_NAME);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
|
||||||
|
const severityFilterName = name as CrimeSeverityFilterName;
|
||||||
|
if (!defaultCrimeSeverityFeatureNames[severityFilterName]) return;
|
||||||
|
queueActiveFilterScroll(severityFilterName, getAddFilterGroupName(severityFilterName));
|
||||||
|
onAddFilter(severityFilterName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
||||||
const filterName = name as PoiFilterName;
|
const filterName = name as PoiFilterName;
|
||||||
if (!defaultPoiFilterFeatureNames[filterName]) return;
|
if (!defaultPoiFilterFeatureNames[filterName]) return;
|
||||||
|
|
@ -528,8 +716,11 @@ export default memo(function Filters({
|
||||||
[
|
[
|
||||||
defaultSchoolFeatureName,
|
defaultSchoolFeatureName,
|
||||||
defaultSpecificCrimeFeatureName,
|
defaultSpecificCrimeFeatureName,
|
||||||
|
defaultCrimeSeverityFeatureNames,
|
||||||
defaultElectionVoteShareFeatureName,
|
defaultElectionVoteShareFeatureName,
|
||||||
defaultEthnicityFeatureName,
|
defaultEthnicityFeatureName,
|
||||||
|
defaultQualificationFeatureName,
|
||||||
|
defaultTenureFeatureName,
|
||||||
defaultPoiFilterFeatureNames,
|
defaultPoiFilterFeatureNames,
|
||||||
getAddFilterGroupName,
|
getAddFilterGroupName,
|
||||||
onAddFilter,
|
onAddFilter,
|
||||||
|
|
@ -686,8 +877,11 @@ export default memo(function Filters({
|
||||||
...features,
|
...features,
|
||||||
schoolMeta,
|
schoolMeta,
|
||||||
specificCrimeMeta,
|
specificCrimeMeta,
|
||||||
|
...Object.values(crimeSeverityMetas),
|
||||||
electionVoteShareMeta,
|
electionVoteShareMeta,
|
||||||
ethnicityMeta,
|
ethnicityMeta,
|
||||||
|
qualificationMeta,
|
||||||
|
tenureMeta,
|
||||||
poiDistanceMeta,
|
poiDistanceMeta,
|
||||||
transportDistanceMeta,
|
transportDistanceMeta,
|
||||||
poiCount2KmMeta,
|
poiCount2KmMeta,
|
||||||
|
|
@ -696,8 +890,11 @@ export default memo(function Filters({
|
||||||
pinnedFeature={pinnedFeature}
|
pinnedFeature={pinnedFeature}
|
||||||
defaultSchoolFeatureName={defaultSchoolFeatureName}
|
defaultSchoolFeatureName={defaultSchoolFeatureName}
|
||||||
defaultSpecificCrimeFeatureName={defaultSpecificCrimeFeatureName}
|
defaultSpecificCrimeFeatureName={defaultSpecificCrimeFeatureName}
|
||||||
|
defaultCrimeSeverityFeatureNames={defaultCrimeSeverityFeatureNames}
|
||||||
defaultElectionVoteShareFeatureName={defaultElectionVoteShareFeatureName}
|
defaultElectionVoteShareFeatureName={defaultElectionVoteShareFeatureName}
|
||||||
defaultEthnicityFeatureName={defaultEthnicityFeatureName}
|
defaultEthnicityFeatureName={defaultEthnicityFeatureName}
|
||||||
|
defaultQualificationFeatureName={defaultQualificationFeatureName}
|
||||||
|
defaultTenureFeatureName={defaultTenureFeatureName}
|
||||||
defaultPoiFilterFeatureNames={defaultPoiFilterFeatureNames}
|
defaultPoiFilterFeatureNames={defaultPoiFilterFeatureNames}
|
||||||
openInfoFeature={openInfoFeature}
|
openInfoFeature={openInfoFeature}
|
||||||
travelTimeEntries={travelTimeEntries}
|
travelTimeEntries={travelTimeEntries}
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,11 @@ import { formatValue } from '../../lib/format';
|
||||||
import { ts } from '../../i18n/server';
|
import { ts } from '../../i18n/server';
|
||||||
import { SCHOOL_FILTER_NAME, getSchoolBackendFeatureName } from '../../lib/school-filter';
|
import { SCHOOL_FILTER_NAME, getSchoolBackendFeatureName } from '../../lib/school-filter';
|
||||||
import { getSpecificCrimeFeatureName } from '../../lib/crime-filter';
|
import { getSpecificCrimeFeatureName } from '../../lib/crime-filter';
|
||||||
|
import { getCrimeSeverityFeatureName } from '../../lib/crime-severity-filter';
|
||||||
import { getElectionVoteShareFeatureName } from '../../lib/election-filter';
|
import { getElectionVoteShareFeatureName } from '../../lib/election-filter';
|
||||||
import { getEthnicityFeatureName } from '../../lib/ethnicity-filter';
|
import { getEthnicityFeatureName } from '../../lib/ethnicity-filter';
|
||||||
|
import { getQualificationFeatureName } from '../../lib/qualification-filter';
|
||||||
|
import { getTenureFeatureName } from '../../lib/tenure-filter';
|
||||||
import {
|
import {
|
||||||
POI_DISTANCE_FILTER_NAME,
|
POI_DISTANCE_FILTER_NAME,
|
||||||
getPoiDistanceFeatureName,
|
getPoiDistanceFeatureName,
|
||||||
|
|
@ -52,14 +55,20 @@ export default memo(function HoverCard({
|
||||||
for (const name of activeFilterNames.slice(0, 4)) {
|
for (const name of activeFilterNames.slice(0, 4)) {
|
||||||
const schoolBackendName = getSchoolBackendFeatureName(name);
|
const schoolBackendName = getSchoolBackendFeatureName(name);
|
||||||
const specificCrimeFeatureName = getSpecificCrimeFeatureName(name);
|
const specificCrimeFeatureName = getSpecificCrimeFeatureName(name);
|
||||||
|
const crimeSeverityFeatureName = getCrimeSeverityFeatureName(name);
|
||||||
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
|
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
|
||||||
const ethnicityFeatureName = getEthnicityFeatureName(name);
|
const ethnicityFeatureName = getEthnicityFeatureName(name);
|
||||||
|
const qualificationFeatureName = getQualificationFeatureName(name);
|
||||||
|
const tenureFeatureName = getTenureFeatureName(name);
|
||||||
const poiDistanceFeatureName = getPoiDistanceFeatureName(name);
|
const poiDistanceFeatureName = getPoiDistanceFeatureName(name);
|
||||||
const backendName =
|
const backendName =
|
||||||
schoolBackendName ??
|
schoolBackendName ??
|
||||||
specificCrimeFeatureName ??
|
specificCrimeFeatureName ??
|
||||||
|
crimeSeverityFeatureName ??
|
||||||
electionVoteShareFeatureName ??
|
electionVoteShareFeatureName ??
|
||||||
ethnicityFeatureName ??
|
ethnicityFeatureName ??
|
||||||
|
qualificationFeatureName ??
|
||||||
|
tenureFeatureName ??
|
||||||
poiDistanceFeatureName ??
|
poiDistanceFeatureName ??
|
||||||
name;
|
name;
|
||||||
const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`];
|
const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`];
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@ vi.mock('react-i18next', () => ({
|
||||||
if (key === 'travel.noBuses') return 'No buses';
|
if (key === 'travel.noBuses') return 'No buses';
|
||||||
if (key === 'areaPane.walk') return 'Walk';
|
if (key === 'areaPane.walk') return 'Walk';
|
||||||
if (key === 'areaPane.cycle') return 'Cycle';
|
if (key === 'areaPane.cycle') return 'Cycle';
|
||||||
|
if (key === 'journey.bus') return 'Bus';
|
||||||
|
if (key === 'journey.lineSuffix') return 'line';
|
||||||
if (key === 'areaPane.viewOnGoogleMaps') return 'View on Google Maps';
|
if (key === 'areaPane.viewOnGoogleMaps') return 'View on Google Maps';
|
||||||
if (key === 'areaPane.noJourneyData') return 'No journey data';
|
if (key === 'areaPane.noJourneyData') return 'No journey data';
|
||||||
return key;
|
return key;
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { useState, useEffect, useMemo } from 'react';
|
import { useState, useEffect, useMemo } from 'react';
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
|
import type { TFunction } from 'i18next';
|
||||||
import type { JourneyLeg } from '../../types';
|
import type { JourneyLeg } from '../../types';
|
||||||
import { resolveTransitVariant, type TravelTimeEntry } from '../../hooks/useTravelTime';
|
import { resolveTransitVariant, type TravelTimeEntry } from '../../hooks/useTravelTime';
|
||||||
import { apiUrl, authHeaders, logNonAbortError } from '../../lib/api';
|
import { apiUrl, authHeaders, logNonAbortError } from '../../lib/api';
|
||||||
|
|
@ -106,15 +107,25 @@ function stripId(label: string): string {
|
||||||
return label.replace(/\s+\([A-Za-z0-9]+\)$/, '');
|
return label.replace(/\s+\([A-Za-z0-9]+\)$/, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
function getRouteDisplay(mode: string): { label: string; color: string; darkText: boolean } {
|
function getRouteDisplay(
|
||||||
|
mode: string,
|
||||||
|
t: TFunction
|
||||||
|
): { label: string; color: string; darkText: boolean } {
|
||||||
const clean = stripId(mode);
|
const clean = stripId(mode);
|
||||||
const known = ROUTE_COLORS[clean];
|
const known = ROUTE_COLORS[clean];
|
||||||
if (known) {
|
if (known) {
|
||||||
const label = NON_TUBE_NAMES.has(clean) || clean.includes('line') ? clean : `${clean} line`;
|
const label =
|
||||||
|
NON_TUBE_NAMES.has(clean) || clean.includes('line')
|
||||||
|
? clean
|
||||||
|
: `${clean} ${t('journey.lineSuffix')}`;
|
||||||
return { label, color: known.color, darkText: !!known.darkText };
|
return { label, color: known.color, darkText: !!known.darkText };
|
||||||
}
|
}
|
||||||
if (/^\d+[A-Za-z]?$/.test(clean.trim())) {
|
if (/^\d+[A-Za-z]?$/.test(clean.trim())) {
|
||||||
return { label: `Bus ${clean}`, color: '#0d9488', darkText: false };
|
return {
|
||||||
|
label: `${t('journey.bus')} ${clean}`,
|
||||||
|
color: '#0d9488',
|
||||||
|
darkText: false,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
return { label: clean, color: '#6b7280', darkText: false };
|
return { label: clean, color: '#6b7280', darkText: false };
|
||||||
}
|
}
|
||||||
|
|
@ -203,7 +214,8 @@ function invertLegs(legs: JourneyLeg[]): JourneyLeg[] {
|
||||||
}
|
}
|
||||||
|
|
||||||
function RouteBadge({ mode }: { mode: string }) {
|
function RouteBadge({ mode }: { mode: string }) {
|
||||||
const { label, color, darkText } = getRouteDisplay(mode);
|
const { t } = useTranslation();
|
||||||
|
const { label, color, darkText } = getRouteDisplay(mode, t);
|
||||||
return (
|
return (
|
||||||
<span
|
<span
|
||||||
className="inline-flex items-center text-[10px] font-bold px-1.5 py-px rounded-sm leading-tight tracking-wide"
|
className="inline-flex items-center text-[10px] font-bold px-1.5 py-px rounded-sm leading-tight tracking-wide"
|
||||||
|
|
@ -242,7 +254,7 @@ function TimelineLeg({ leg, isLast }: { leg: JourneyLeg; isLast: boolean }) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { color } = getRouteDisplay(leg.mode);
|
const { color } = getRouteDisplay(leg.mode, t);
|
||||||
return (
|
return (
|
||||||
<div className="flex">
|
<div className="flex">
|
||||||
<div className="flex flex-col items-center w-4 mr-2">
|
<div className="flex flex-col items-center w-4 mr-2">
|
||||||
|
|
|
||||||
|
|
@ -19,15 +19,21 @@ function formatListingHeadline(listing: ActualListing, t: TFunction): string | n
|
||||||
|
|
||||||
export const ListingPopupSingleContent = memo(function ListingPopupSingleContent({
|
export const ListingPopupSingleContent = memo(function ListingPopupSingleContent({
|
||||||
listing,
|
listing,
|
||||||
|
clickedUrls,
|
||||||
|
onOpen,
|
||||||
}: {
|
}: {
|
||||||
listing: ActualListing;
|
listing: ActualListing;
|
||||||
|
clickedUrls: Set<string>;
|
||||||
|
onOpen: (url: string) => void;
|
||||||
}) {
|
}) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const visited = clickedUrls.has(listing.listing_url);
|
||||||
return (
|
return (
|
||||||
<a
|
<a
|
||||||
href={listing.listing_url}
|
href={listing.listing_url}
|
||||||
target="_blank"
|
target="_blank"
|
||||||
rel="noopener noreferrer"
|
rel="noopener noreferrer"
|
||||||
|
onClick={() => onOpen(listing.listing_url)}
|
||||||
className="block px-3 py-2"
|
className="block px-3 py-2"
|
||||||
>
|
>
|
||||||
{listing.asking_price != null && (
|
{listing.asking_price != null && (
|
||||||
|
|
@ -57,7 +63,7 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
|
||||||
)}
|
)}
|
||||||
{listing.floor_area_sqm != null && (
|
{listing.floor_area_sqm != null && (
|
||||||
<div className="text-[11px] text-warm-500 dark:text-warm-400 mt-0.5">
|
<div className="text-[11px] text-warm-500 dark:text-warm-400 mt-0.5">
|
||||||
{Math.round(listing.floor_area_sqm)} sqm
|
{Math.round(listing.floor_area_sqm)} {t('common.sqm')}
|
||||||
{listing.asking_price_per_sqm != null
|
{listing.asking_price_per_sqm != null
|
||||||
? ` · £${Math.round(listing.asking_price_per_sqm).toLocaleString()}/sqm`
|
? ` · £${Math.round(listing.asking_price_per_sqm).toLocaleString()}/sqm`
|
||||||
: ''}
|
: ''}
|
||||||
|
|
@ -72,8 +78,9 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
|
||||||
))}
|
))}
|
||||||
</ul>
|
</ul>
|
||||||
)}
|
)}
|
||||||
<div className="mt-1.5 text-[11px] text-teal-600 dark:text-teal-400 font-medium">
|
<div className="mt-1.5 flex items-center gap-1.5 text-[11px] font-medium">
|
||||||
Open listing ↗
|
{visited && <span className="text-violet-600 dark:text-violet-400">✓ {t('listing.viewed')}</span>}
|
||||||
|
<span className="text-teal-600 dark:text-teal-400">{t('listing.openListing')} ↗</span>
|
||||||
</div>
|
</div>
|
||||||
</a>
|
</a>
|
||||||
);
|
);
|
||||||
|
|
@ -82,9 +89,13 @@ export const ListingPopupSingleContent = memo(function ListingPopupSingleContent
|
||||||
export const ListingClusterPopupContent = memo(function ListingClusterPopupContent({
|
export const ListingClusterPopupContent = memo(function ListingClusterPopupContent({
|
||||||
count,
|
count,
|
||||||
listings,
|
listings,
|
||||||
|
clickedUrls,
|
||||||
|
onOpen,
|
||||||
}: {
|
}: {
|
||||||
count: number;
|
count: number;
|
||||||
listings: ActualListing[];
|
listings: ActualListing[];
|
||||||
|
clickedUrls: Set<string>;
|
||||||
|
onOpen: (url: string) => void;
|
||||||
}) {
|
}) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const visibleCount = listings.length;
|
const visibleCount = listings.length;
|
||||||
|
|
@ -92,32 +103,41 @@ export const ListingClusterPopupContent = memo(function ListingClusterPopupConte
|
||||||
<div>
|
<div>
|
||||||
<div className="border-b border-warm-200 px-3 py-2 dark:border-warm-700">
|
<div className="border-b border-warm-200 px-3 py-2 dark:border-warm-700">
|
||||||
<div className="text-base font-bold text-red-600 dark:text-red-400">
|
<div className="text-base font-bold text-red-600 dark:text-red-400">
|
||||||
{count.toLocaleString()} listings
|
{count.toLocaleString()} {t('listing.listings')}
|
||||||
</div>
|
</div>
|
||||||
<div className="text-[11px] text-warm-500 dark:text-warm-400">
|
<div className="text-[11px] text-warm-500 dark:text-warm-400">
|
||||||
{visibleCount > 0
|
{visibleCount > 0
|
||||||
? `Showing ${visibleCount.toLocaleString()} of ${count.toLocaleString()}`
|
? t('listing.showingOf', { visible: visibleCount, count })
|
||||||
: 'Grouped near this map position'}
|
: t('listing.groupedNear')}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{visibleCount > 0 && (
|
{visibleCount > 0 && (
|
||||||
<div className="max-h-80 overflow-y-auto py-1">
|
<div className="max-h-80 overflow-y-auto py-1">
|
||||||
{listings.map((listing, idx) => {
|
{listings.map((listing, idx) => {
|
||||||
const headline = formatListingHeadline(listing, t);
|
const headline = formatListingHeadline(listing, t);
|
||||||
|
const visited = clickedUrls.has(listing.listing_url);
|
||||||
return (
|
return (
|
||||||
<a
|
<a
|
||||||
key={`${listing.listing_url}-${idx}`}
|
key={`${listing.listing_url}-${idx}`}
|
||||||
href={listing.listing_url}
|
href={listing.listing_url}
|
||||||
target="_blank"
|
target="_blank"
|
||||||
rel="noopener noreferrer"
|
rel="noopener noreferrer"
|
||||||
|
onClick={() => onOpen(listing.listing_url)}
|
||||||
className="block border-b border-warm-100 px-3 py-2 last:border-b-0 hover:bg-warm-50 dark:border-warm-700 dark:hover:bg-warm-700/60"
|
className="block border-b border-warm-100 px-3 py-2 last:border-b-0 hover:bg-warm-50 dark:border-warm-700 dark:hover:bg-warm-700/60"
|
||||||
>
|
>
|
||||||
<div className="flex items-start justify-between gap-3">
|
<div className="flex items-start justify-between gap-3">
|
||||||
<div className="min-w-0">
|
<div className="min-w-0">
|
||||||
<div className="text-sm font-semibold text-teal-700 dark:text-teal-300">
|
<div className="flex items-center gap-1.5">
|
||||||
{listing.asking_price != null
|
<span className="text-sm font-semibold text-teal-700 dark:text-teal-300">
|
||||||
? formatListingPrice(listing.asking_price)
|
{listing.asking_price != null
|
||||||
: 'Listing'}
|
? formatListingPrice(listing.asking_price)
|
||||||
|
: t('listing.listing')}
|
||||||
|
</span>
|
||||||
|
{visited && (
|
||||||
|
<span className="shrink-0 text-[10px] font-medium text-violet-600 dark:text-violet-400">
|
||||||
|
✓ Viewed
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
{headline && (
|
{headline && (
|
||||||
<div className="mt-0.5 truncate text-xs text-warm-700 dark:text-warm-200">
|
<div className="mt-0.5 truncate text-xs text-warm-700 dark:text-warm-200">
|
||||||
|
|
|
||||||
|
|
@ -452,6 +452,8 @@ export default memo(function Map({
|
||||||
visiblePois,
|
visiblePois,
|
||||||
listingPopup,
|
listingPopup,
|
||||||
clearListingPopup,
|
clearListingPopup,
|
||||||
|
clickedListingUrls,
|
||||||
|
markListingClicked,
|
||||||
developmentPopup,
|
developmentPopup,
|
||||||
clearDevelopmentPopup,
|
clearDevelopmentPopup,
|
||||||
hoverPosition,
|
hoverPosition,
|
||||||
|
|
@ -696,11 +698,17 @@ export default memo(function Map({
|
||||||
<CloseIcon className="w-3 h-3" />
|
<CloseIcon className="w-3 h-3" />
|
||||||
</button>
|
</button>
|
||||||
{listingPopup.mode === 'single' ? (
|
{listingPopup.mode === 'single' ? (
|
||||||
<ListingPopupSingleContent listing={listingPopup.listing} />
|
<ListingPopupSingleContent
|
||||||
|
listing={listingPopup.listing}
|
||||||
|
clickedUrls={clickedListingUrls}
|
||||||
|
onOpen={markListingClicked}
|
||||||
|
/>
|
||||||
) : (
|
) : (
|
||||||
<ListingClusterPopupContent
|
<ListingClusterPopupContent
|
||||||
count={listingPopup.count}
|
count={listingPopup.count}
|
||||||
listings={listingPopup.listings}
|
listings={listingPopup.listings}
|
||||||
|
clickedUrls={clickedListingUrls}
|
||||||
|
onOpen={markListingClicked}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { Component, Fragment, type ReactNode } from 'react';
|
import { Component, Fragment, type ReactNode } from 'react';
|
||||||
import * as Sentry from '@sentry/react';
|
import * as Sentry from '@sentry/react';
|
||||||
|
|
||||||
|
import i18n from '../../i18n';
|
||||||
import { MapFallback } from './map-page/Fallbacks';
|
import { MapFallback } from './map-page/Fallbacks';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -113,17 +114,19 @@ export class MapErrorBoundary extends Component<MapErrorBoundaryProps, MapErrorB
|
||||||
<div className="flex h-full w-full items-center justify-center bg-warm-100 px-6 text-center dark:bg-navy-950">
|
<div className="flex h-full w-full items-center justify-center bg-warm-100 px-6 text-center dark:bg-navy-950">
|
||||||
<div>
|
<div>
|
||||||
<h2 className="text-lg font-semibold text-warm-900 dark:text-warm-100">
|
<h2 className="text-lg font-semibold text-warm-900 dark:text-warm-100">
|
||||||
The map ran into a problem
|
{i18n.t('map.error.heading', { defaultValue: 'The map ran into a problem' })}
|
||||||
</h2>
|
</h2>
|
||||||
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
|
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
|
||||||
This can happen when your browser's graphics context is interrupted.
|
{i18n.t('map.error.body', {
|
||||||
|
defaultValue: 'This can happen when your browser’s graphics context is interrupted.',
|
||||||
|
})}
|
||||||
</p>
|
</p>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={this.handleManualRetry}
|
onClick={this.handleManualRetry}
|
||||||
className="mt-4 rounded-lg bg-teal-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-teal-700 dark:bg-teal-500 dark:hover:bg-teal-400"
|
className="mt-4 rounded-lg bg-teal-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-teal-700 dark:bg-teal-500 dark:hover:bg-teal-400"
|
||||||
>
|
>
|
||||||
Reload the map
|
{i18n.t('map.error.reload', { defaultValue: 'Reload the map' })}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -407,6 +407,7 @@ export default function MapPage({
|
||||||
handlePropertiesTabClick,
|
handlePropertiesTabClick,
|
||||||
handleLoadMoreProperties,
|
handleLoadMoreProperties,
|
||||||
handleCloseSelection,
|
handleCloseSelection,
|
||||||
|
loadCrimeRecords,
|
||||||
selectedPostcodeGeometry,
|
selectedPostcodeGeometry,
|
||||||
handleLocationSearch,
|
handleLocationSearch,
|
||||||
handleCurrentLocationSearch,
|
handleCurrentLocationSearch,
|
||||||
|
|
@ -806,6 +807,7 @@ export default function MapPage({
|
||||||
shareCode={shareCode}
|
shareCode={shareCode}
|
||||||
isGroupExpanded={isAreaGroupExpanded}
|
isGroupExpanded={isAreaGroupExpanded}
|
||||||
onToggleGroup={toggleAreaGroup}
|
onToggleGroup={toggleAreaGroup}
|
||||||
|
onLoadCrimeRecords={loadCrimeRecords}
|
||||||
scrollTopRef={areaPaneScrollTopRef}
|
scrollTopRef={areaPaneScrollTopRef}
|
||||||
scrollRestoreKey={
|
scrollRestoreKey={
|
||||||
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
|
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
|
||||||
|
|
@ -823,6 +825,7 @@ export default function MapPage({
|
||||||
hexagonLocation,
|
hexagonLocation,
|
||||||
isAreaGroupExpanded,
|
isAreaGroupExpanded,
|
||||||
loadingAreaStats,
|
loadingAreaStats,
|
||||||
|
loadCrimeRecords,
|
||||||
selectedHexagon,
|
selectedHexagon,
|
||||||
setAreaStatsUseFilters,
|
setAreaStatsUseFilters,
|
||||||
shareCode,
|
shareCode,
|
||||||
|
|
|
||||||
|
|
@ -105,7 +105,11 @@ export default function MobileDrawer({
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
data-tutorial="right-pane"
|
data-tutorial="right-pane"
|
||||||
className="pointer-events-none fixed inset-0 z-50 flex flex-col"
|
// Pin to the dynamic viewport (100dvh) anchored at the top instead of
|
||||||
|
// `inset-0`. On iOS Safari a fixed element with `bottom: 0` is laid out
|
||||||
|
// against the large viewport, so when the bottom toolbar shows the panel's
|
||||||
|
// bottom (and the last collapsible group) hides behind it and can't be tapped.
|
||||||
|
className="pointer-events-none fixed inset-x-0 top-0 z-50 flex h-[100dvh] flex-col"
|
||||||
>
|
>
|
||||||
<div className="h-[10%] shrink-0" aria-hidden="true" />
|
<div className="h-[10%] shrink-0" aria-hidden="true" />
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next';
|
||||||
import { BASEMAPS, type BasemapId } from '../../lib/basemaps';
|
import { BASEMAPS, type BasemapId } from '../../lib/basemaps';
|
||||||
import { OVERLAYS, type OverlayDefinition, type OverlayId } from '../../lib/overlays';
|
import { OVERLAYS, type OverlayDefinition, type OverlayId } from '../../lib/overlays';
|
||||||
import { CRIME_TYPES, CRIME_TYPE_VALUES } from '../../lib/crime-types';
|
import { CRIME_TYPES, CRIME_TYPE_VALUES } from '../../lib/crime-types';
|
||||||
|
import { tDynamic } from '../../i18n';
|
||||||
import { PillToggle } from '../ui/PillToggle';
|
import { PillToggle } from '../ui/PillToggle';
|
||||||
import { IconButton } from '../ui/IconButton';
|
import { IconButton } from '../ui/IconButton';
|
||||||
import { Slider } from '../ui/Slider';
|
import { Slider } from '../ui/Slider';
|
||||||
|
|
@ -12,6 +13,18 @@ import { colorOpacityToPercent, normalizeColorOpacity } from '../../lib/color-op
|
||||||
|
|
||||||
const CRIME_OVERLAY_ID: OverlayId = 'crime-hotspots';
|
const CRIME_OVERLAY_ID: OverlayId = 'crime-hotspots';
|
||||||
|
|
||||||
|
// Maps the kebab-case overlay id to its camelCase i18n key under `overlays.*`.
|
||||||
|
const OVERLAY_I18N_KEY: Record<OverlayId, string> = {
|
||||||
|
noise: 'noise',
|
||||||
|
'crime-hotspots': 'crimeHotspots',
|
||||||
|
'trees-outside-woodlands': 'treesOutsideWoodlands',
|
||||||
|
'property-borders': 'propertyBorders',
|
||||||
|
'new-developments': 'newDevelopments',
|
||||||
|
};
|
||||||
|
|
||||||
|
const overlayLabel = (id: OverlayId) => tDynamic(`overlays.${OVERLAY_I18N_KEY[id]}.label`);
|
||||||
|
const overlayDetail = (id: OverlayId) => tDynamic(`overlays.${OVERLAY_I18N_KEY[id]}.detail`);
|
||||||
|
|
||||||
interface OverlayPaneProps {
|
interface OverlayPaneProps {
|
||||||
selectedOverlays: Set<OverlayId>;
|
selectedOverlays: Set<OverlayId>;
|
||||||
onOverlaysChange: (overlays: Set<OverlayId>) => void;
|
onOverlaysChange: (overlays: Set<OverlayId>) => void;
|
||||||
|
|
@ -78,7 +91,7 @@ export default function OverlayPane({
|
||||||
<div className="flex-shrink-0 px-3 pt-3 pb-2">
|
<div className="flex-shrink-0 px-3 pt-3 pb-2">
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2">
|
||||||
<span className="text-xs font-semibold text-warm-500 dark:text-warm-400 uppercase tracking-wide">
|
<span className="text-xs font-semibold text-warm-500 dark:text-warm-400 uppercase tracking-wide">
|
||||||
Overlays
|
{t('overlays.heading')}
|
||||||
</span>
|
</span>
|
||||||
<span className="text-xs text-warm-400 dark:text-warm-500">
|
<span className="text-xs text-warm-400 dark:text-warm-500">
|
||||||
{selectedOverlays.size}/{OVERLAYS.length}
|
{selectedOverlays.size}/{OVERLAYS.length}
|
||||||
|
|
@ -88,13 +101,13 @@ export default function OverlayPane({
|
||||||
onClick={selectNone}
|
onClick={selectNone}
|
||||||
className="rounded border border-warm-300 px-2 py-0.5 text-xs text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
className="rounded border border-warm-300 px-2 py-0.5 text-xs text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
||||||
>
|
>
|
||||||
None
|
{t('common.none')}
|
||||||
</button>
|
</button>
|
||||||
{onClose && (
|
{onClose && (
|
||||||
<button
|
<button
|
||||||
onClick={onClose}
|
onClick={onClose}
|
||||||
className="ml-1 p-0.5 text-warm-400 hover:text-warm-700 dark:hover:text-warm-300"
|
className="ml-1 p-0.5 text-warm-400 hover:text-warm-700 dark:hover:text-warm-300"
|
||||||
title="Close"
|
title={t('common.close')}
|
||||||
>
|
>
|
||||||
<CloseIcon className="h-4 w-4" />
|
<CloseIcon className="h-4 w-4" />
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -106,8 +119,7 @@ export default function OverlayPane({
|
||||||
role="alert"
|
role="alert"
|
||||||
className="mt-2 rounded border border-amber-300 bg-amber-50 px-2 py-1.5 text-xs text-amber-800 dark:border-amber-700/60 dark:bg-amber-900/30 dark:text-amber-200"
|
className="mt-2 rounded border border-amber-300 bg-amber-50 px-2 py-1.5 text-xs text-amber-800 dark:border-amber-700/60 dark:bg-amber-900/30 dark:text-amber-200"
|
||||||
>
|
>
|
||||||
Zoom in further to see the selected{' '}
|
{t('overlays.zoomWarning', { count: selectedOverlays.size })}
|
||||||
{selectedOverlays.size === 1 ? 'overlay' : 'overlays'}.
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -115,13 +127,13 @@ export default function OverlayPane({
|
||||||
<div className="min-h-0 space-y-4 overflow-y-auto overscroll-contain border-t border-warm-200 px-3 py-3 dark:border-warm-700">
|
<div className="min-h-0 space-y-4 overflow-y-auto overscroll-contain border-t border-warm-200 px-3 py-3 dark:border-warm-700">
|
||||||
<div>
|
<div>
|
||||||
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
||||||
Base map
|
{t('overlays.baseMap')}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex flex-wrap gap-1.5">
|
<div className="flex flex-wrap gap-1.5">
|
||||||
{BASEMAPS.map((option) => (
|
{BASEMAPS.map((option) => (
|
||||||
<PillToggle
|
<PillToggle
|
||||||
key={option.id}
|
key={option.id}
|
||||||
label={option.label}
|
label={tDynamic(`map.basemap.${option.id}`)}
|
||||||
active={basemap === option.id}
|
active={basemap === option.id}
|
||||||
onClick={() => onBasemapChange(option.id)}
|
onClick={() => onBasemapChange(option.id)}
|
||||||
size="sm"
|
size="sm"
|
||||||
|
|
@ -133,7 +145,7 @@ export default function OverlayPane({
|
||||||
<div>
|
<div>
|
||||||
<div className="mb-2 flex items-center justify-between gap-2">
|
<div className="mb-2 flex items-center justify-between gap-2">
|
||||||
<span className="text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
<span className="text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
||||||
Colour opacity
|
{t('overlays.colourOpacity')}
|
||||||
</span>
|
</span>
|
||||||
<span className="text-[10px] font-medium tabular-nums text-warm-500 dark:text-warm-400">
|
<span className="text-[10px] font-medium tabular-nums text-warm-500 dark:text-warm-400">
|
||||||
{colorOpacityPercent}%
|
{colorOpacityPercent}%
|
||||||
|
|
@ -145,27 +157,27 @@ export default function OverlayPane({
|
||||||
step={5}
|
step={5}
|
||||||
value={[colorOpacityPercent]}
|
value={[colorOpacityPercent]}
|
||||||
onValueChange={handleColorOpacityChange}
|
onValueChange={handleColorOpacityChange}
|
||||||
aria-label="Colour opacity"
|
aria-label={t('overlays.colourOpacity')}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div>
|
<div>
|
||||||
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
<div className="mb-2 text-[10px] font-semibold uppercase tracking-wide text-warm-400 dark:text-warm-500">
|
||||||
Data overlays
|
{t('overlays.dataOverlays')}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex flex-wrap gap-1.5">
|
<div className="flex flex-wrap gap-1.5">
|
||||||
{OVERLAYS.map((overlay) => (
|
{OVERLAYS.map((overlay) => (
|
||||||
<div key={overlay.id} className="inline-flex items-center gap-0.5">
|
<div key={overlay.id} className="inline-flex items-center gap-0.5">
|
||||||
<PillToggle
|
<PillToggle
|
||||||
label={overlay.label}
|
label={overlayLabel(overlay.id)}
|
||||||
active={selectedOverlays.has(overlay.id)}
|
active={selectedOverlays.has(overlay.id)}
|
||||||
onClick={() => toggleOverlay(overlay.id)}
|
onClick={() => toggleOverlay(overlay.id)}
|
||||||
size="sm"
|
size="sm"
|
||||||
/>
|
/>
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={() => setInfoOverlay(overlay)}
|
onClick={() => setInfoOverlay(overlay)}
|
||||||
title={`About ${overlay.label}`}
|
title={t('overlays.about', { name: overlayLabel(overlay.id) })}
|
||||||
ariaLabel={`About ${overlay.label}`}
|
ariaLabel={t('overlays.about', { name: overlayLabel(overlay.id) })}
|
||||||
>
|
>
|
||||||
<InfoIcon className="h-3.5 w-3.5" />
|
<InfoIcon className="h-3.5 w-3.5" />
|
||||||
</IconButton>
|
</IconButton>
|
||||||
|
|
@ -188,13 +200,13 @@ export default function OverlayPane({
|
||||||
onClick={selectAllCrimeTypes}
|
onClick={selectAllCrimeTypes}
|
||||||
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
||||||
>
|
>
|
||||||
All
|
{t('common.all')}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={selectNoCrimeTypes}
|
onClick={selectNoCrimeTypes}
|
||||||
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
className="rounded border border-warm-300 px-1.5 py-0.5 text-[10px] text-warm-600 hover:bg-warm-50 dark:border-warm-700 dark:text-warm-400 dark:hover:bg-warm-700"
|
||||||
>
|
>
|
||||||
None
|
{t('common.none')}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -210,7 +222,7 @@ export default function OverlayPane({
|
||||||
onChange={() => toggleCrimeType(crime.value)}
|
onChange={() => toggleCrimeType(crime.value)}
|
||||||
className="h-3.5 w-3.5 shrink-0 rounded border-warm-300 text-amber-600 focus:ring-1 focus:ring-amber-500 dark:border-warm-600 dark:bg-warm-800"
|
className="h-3.5 w-3.5 shrink-0 rounded border-warm-300 text-amber-600 focus:ring-1 focus:ring-amber-500 dark:border-warm-600 dark:bg-warm-800"
|
||||||
/>
|
/>
|
||||||
<span>{crime.label}</span>
|
<span>{tDynamic(crime.labelKey)}</span>
|
||||||
</label>
|
</label>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -219,9 +231,9 @@ export default function OverlayPane({
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{infoOverlay && (
|
{infoOverlay && (
|
||||||
<InfoPopup title={infoOverlay.label} onClose={() => setInfoOverlay(null)}>
|
<InfoPopup title={overlayLabel(infoOverlay.id)} onClose={() => setInfoOverlay(null)}>
|
||||||
<p className="text-sm text-warm-700 dark:text-warm-300 leading-relaxed">
|
<p className="text-sm text-warm-700 dark:text-warm-300 leading-relaxed">
|
||||||
{infoOverlay.detail}
|
{overlayDetail(infoOverlay.id)}
|
||||||
</p>
|
</p>
|
||||||
</InfoPopup>
|
</InfoPopup>
|
||||||
)}
|
)}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
import { memo } from 'react';
|
import { memo } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
import type { TFunction } from 'i18next';
|
||||||
|
|
||||||
import type { SchoolMetadata } from '../../types';
|
import type { SchoolMetadata } from '../../types';
|
||||||
import { POI_GROUP_COLORS } from '../../lib/consts';
|
import { POI_GROUP_COLORS } from '../../lib/consts';
|
||||||
|
|
@ -32,7 +34,7 @@ function normalizeSchoolWebsiteUrl(raw: string): string | null {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderSchoolMetadata(school: SchoolMetadata) {
|
function renderSchoolMetadata(school: SchoolMetadata, t: TFunction) {
|
||||||
// First line collects the headline classification (phase, type, religious
|
// First line collects the headline classification (phase, type, religious
|
||||||
// character) so the popup is scannable even when most fields are absent.
|
// character) so the popup is scannable even when most fields are absent.
|
||||||
const headline: string[] = [];
|
const headline: string[] = [];
|
||||||
|
|
@ -41,11 +43,11 @@ function renderSchoolMetadata(school: SchoolMetadata) {
|
||||||
|
|
||||||
const pupilsLine =
|
const pupilsLine =
|
||||||
school.pupils !== undefined && school.capacity !== undefined
|
school.pupils !== undefined && school.capacity !== undefined
|
||||||
? `${school.pupils.toLocaleString()} / ${school.capacity.toLocaleString()} pupils`
|
? `${school.pupils.toLocaleString()} / ${school.capacity.toLocaleString()} ${t('poiPopup.school.pupils')}`
|
||||||
: school.pupils !== undefined
|
: school.pupils !== undefined
|
||||||
? `${school.pupils.toLocaleString()} pupils`
|
? `${school.pupils.toLocaleString()} ${t('poiPopup.school.pupils')}`
|
||||||
: school.capacity !== undefined
|
: school.capacity !== undefined
|
||||||
? `Capacity ${school.capacity.toLocaleString()}`
|
? `${t('poiPopup.school.capacity')} ${school.capacity.toLocaleString()}`
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
const websiteUrl = school.website ? normalizeSchoolWebsiteUrl(school.website) : null;
|
const websiteUrl = school.website ? normalizeSchoolWebsiteUrl(school.website) : null;
|
||||||
|
|
@ -54,69 +56,69 @@ function renderSchoolMetadata(school: SchoolMetadata) {
|
||||||
<dl className="mt-2 grid grid-cols-[auto_1fr] gap-x-2 gap-y-0.5 text-xs text-warm-600 dark:text-warm-300">
|
<dl className="mt-2 grid grid-cols-[auto_1fr] gap-x-2 gap-y-0.5 text-xs text-warm-600 dark:text-warm-300">
|
||||||
{headline.length > 0 && (
|
{headline.length > 0 && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Type</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.type')}</dt>
|
||||||
<dd className="dark:text-warm-200">{headline.join(' · ')}</dd>
|
<dd className="dark:text-warm-200">{headline.join(' · ')}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.age_range && (
|
{school.age_range && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Ages</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.ages')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.age_range}</dd>
|
<dd className="dark:text-warm-200">{school.age_range}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.gender && school.gender !== 'Mixed' && (
|
{school.gender && school.gender !== 'Mixed' && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Gender</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.gender')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.gender}</dd>
|
<dd className="dark:text-warm-200">{school.gender}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{pupilsLine && (
|
{pupilsLine && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Pupils</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.pupilsLabel')}</dt>
|
||||||
<dd className="dark:text-warm-200">{pupilsLine}</dd>
|
<dd className="dark:text-warm-200">{pupilsLine}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.fsm_percent !== undefined && (
|
{school.fsm_percent !== undefined && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Free meal</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.freeMeal')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.fsm_percent.toFixed(1)}%</dd>
|
<dd className="dark:text-warm-200">{school.fsm_percent.toFixed(1)}%</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.ofsted_rating && (
|
{school.ofsted_rating && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Ofsted</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.ofsted')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.ofsted_rating}</dd>
|
<dd className="dark:text-warm-200">{school.ofsted_rating}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.sixth_form === 'Has a sixth form' && (
|
{school.sixth_form === 'Has a sixth form' && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Sixth form</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.sixthForm')}</dt>
|
||||||
<dd className="dark:text-warm-200">Yes</dd>
|
<dd className="dark:text-warm-200">{t('common.yes')}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.religious_character &&
|
{school.religious_character &&
|
||||||
school.religious_character !== 'Does not apply' &&
|
school.religious_character !== 'Does not apply' &&
|
||||||
school.religious_character !== 'None' && (
|
school.religious_character !== 'None' && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Religion</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.religion')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.religious_character}</dd>
|
<dd className="dark:text-warm-200">{school.religious_character}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.admissions_policy && (
|
{school.admissions_policy && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Admissions</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.admissions')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.admissions_policy}</dd>
|
<dd className="dark:text-warm-200">{school.admissions_policy}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.trust && (
|
{school.trust && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Trust</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.trust')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.trust}</dd>
|
<dd className="dark:text-warm-200">{school.trust}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{(school.address || school.postcode) && (
|
{(school.address || school.postcode) && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Address</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.address')}</dt>
|
||||||
<dd className="dark:text-warm-200">
|
<dd className="dark:text-warm-200">
|
||||||
{[school.address, school.postcode].filter(Boolean).join(', ')}
|
{[school.address, school.postcode].filter(Boolean).join(', ')}
|
||||||
</dd>
|
</dd>
|
||||||
|
|
@ -124,19 +126,19 @@ function renderSchoolMetadata(school: SchoolMetadata) {
|
||||||
)}
|
)}
|
||||||
{school.local_authority && (
|
{school.local_authority && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">LA</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.localAuthority')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.local_authority}</dd>
|
<dd className="dark:text-warm-200">{school.local_authority}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{school.head_name && (
|
{school.head_name && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Head</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.head')}</dt>
|
||||||
<dd className="dark:text-warm-200">{school.head_name}</dd>
|
<dd className="dark:text-warm-200">{school.head_name}</dd>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
{websiteUrl && (
|
{websiteUrl && (
|
||||||
<>
|
<>
|
||||||
<dt className="text-warm-500 dark:text-warm-400">Website</dt>
|
<dt className="text-warm-500 dark:text-warm-400">{t('poiPopup.school.website')}</dt>
|
||||||
<dd className="truncate">
|
<dd className="truncate">
|
||||||
<a
|
<a
|
||||||
href={websiteUrl}
|
href={websiteUrl}
|
||||||
|
|
@ -158,6 +160,7 @@ export const PoiPopupCardContent = memo(function PoiPopupCardContent({
|
||||||
}: {
|
}: {
|
||||||
poi: PoiPopupCardData;
|
poi: PoiPopupCardData;
|
||||||
}) {
|
}) {
|
||||||
|
const { t } = useTranslation();
|
||||||
return (
|
return (
|
||||||
<div className="px-3 py-2 max-w-[280px]">
|
<div className="px-3 py-2 max-w-[280px]">
|
||||||
<div className="flex items-center gap-2">
|
<div className="flex items-center gap-2">
|
||||||
|
|
@ -182,7 +185,7 @@ export const PoiPopupCardContent = memo(function PoiPopupCardContent({
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{poi.school && renderSchoolMetadata(poi.school)}
|
{poi.school && renderSchoolMetadata(poi.school, t)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,11 @@ export function PropertiesPane({
|
||||||
return (
|
return (
|
||||||
<div className="relative flex h-full flex-col">
|
<div className="relative flex h-full flex-col">
|
||||||
<IndeterminateProgressBar show={loading && properties.length > 0} />
|
<IndeterminateProgressBar show={loading && properties.length > 0} />
|
||||||
<div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto">
|
<div
|
||||||
|
ref={scrollRef}
|
||||||
|
onScroll={onScroll}
|
||||||
|
className="flex-1 overflow-y-auto pb-[env(safe-area-inset-bottom)]"
|
||||||
|
>
|
||||||
{showInfo && (
|
{showInfo && (
|
||||||
<InfoPopup
|
<InfoPopup
|
||||||
title={t('propertyCard.propertyData')}
|
title={t('propertyCard.propertyData')}
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ interface StackedBarChartProps {
|
||||||
/** Strip common suffixes/prefixes to produce short legend labels */
|
/** Strip common suffixes/prefixes to produce short legend labels */
|
||||||
function shortenLabel(name: string): string {
|
function shortenLabel(name: string): string {
|
||||||
return name
|
return name
|
||||||
|
.replace(/ \(\/yr, \d+y\)$/, '')
|
||||||
.replace(' (avg/yr)', '')
|
.replace(' (avg/yr)', '')
|
||||||
.replace(/^% /, '')
|
.replace(/^% /, '')
|
||||||
.replace('and sexual offences', '')
|
.replace('and sexual offences', '')
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,32 @@ import { useMemo } from 'react';
|
||||||
import type { FeatureFilters, FeatureMeta } from '../../../types';
|
import type { FeatureFilters, FeatureMeta } from '../../../types';
|
||||||
import type { PercentileScale } from '../../../lib/format';
|
import type { PercentileScale } from '../../../lib/format';
|
||||||
import { groupFeaturesByCategory } from '../../../lib/features';
|
import { groupFeaturesByCategory } from '../../../lib/features';
|
||||||
import { getSpecificCrimeFeatureName, isSpecificCrimeFilterName } from '../../../lib/crime-filter';
|
import {
|
||||||
|
SPECIFIC_CRIME_VARIANT_CONFIG,
|
||||||
|
getSpecificCrimeFeatureName,
|
||||||
|
isSpecificCrimeFilterName,
|
||||||
|
} from '../../../lib/crime-filter';
|
||||||
|
import {
|
||||||
|
getCrimeSeverityFeatureName,
|
||||||
|
getCrimeSeverityFilterName,
|
||||||
|
getCrimeSeverityVariantConfig,
|
||||||
|
isCrimeSeverityFilterName,
|
||||||
|
} from '../../../lib/crime-severity-filter';
|
||||||
import {
|
import {
|
||||||
getElectionVoteShareFeatureName,
|
getElectionVoteShareFeatureName,
|
||||||
isElectionVoteShareFilterName,
|
isElectionVoteShareFilterName,
|
||||||
} from '../../../lib/election-filter';
|
} from '../../../lib/election-filter';
|
||||||
import { getEthnicityFeatureName, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
|
import { getEthnicityFeatureName, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
|
||||||
|
import {
|
||||||
|
QUALIFICATION_VARIANT_CONFIG,
|
||||||
|
getQualificationFeatureName,
|
||||||
|
isQualificationFilterName,
|
||||||
|
} from '../../../lib/qualification-filter';
|
||||||
|
import {
|
||||||
|
TENURE_VARIANT_CONFIG,
|
||||||
|
getTenureFeatureName,
|
||||||
|
isTenureFilterName,
|
||||||
|
} from '../../../lib/tenure-filter';
|
||||||
import { getSchoolBackendFeatureName, isSchoolFilterName } from '../../../lib/school-filter';
|
import { getSchoolBackendFeatureName, isSchoolFilterName } from '../../../lib/school-filter';
|
||||||
import {
|
import {
|
||||||
getPoiDistanceFeatureName,
|
getPoiDistanceFeatureName,
|
||||||
|
|
@ -18,7 +38,7 @@ import type { TravelTimeEntry } from '../../../hooks/useTravelTime';
|
||||||
import { EthnicityFilterCard } from './EthnicityFilterCard';
|
import { EthnicityFilterCard } from './EthnicityFilterCard';
|
||||||
import { PoiDistanceFilterCard } from './PoiDistanceFilterCard';
|
import { PoiDistanceFilterCard } from './PoiDistanceFilterCard';
|
||||||
import { SchoolFilterCard } from './SchoolFilterCard';
|
import { SchoolFilterCard } from './SchoolFilterCard';
|
||||||
import { SpecificCrimeFilterCard } from './SpecificCrimeFilterCard';
|
import { VariantFilterCard } from './VariantFilterCard';
|
||||||
import { ElectionVoteShareFilterCard } from './ElectionVoteShareFilterCard';
|
import { ElectionVoteShareFilterCard } from './ElectionVoteShareFilterCard';
|
||||||
import { EnumFeatureFilterCard } from './EnumFeatureFilterCard';
|
import { EnumFeatureFilterCard } from './EnumFeatureFilterCard';
|
||||||
import { NumericFeatureFilterCard } from './NumericFeatureFilterCard';
|
import { NumericFeatureFilterCard } from './NumericFeatureFilterCard';
|
||||||
|
|
@ -40,6 +60,10 @@ interface ActiveFilterListProps {
|
||||||
destinationDropdownPortal: boolean;
|
destinationDropdownPortal: boolean;
|
||||||
isGroupExpanded: (name: string) => boolean;
|
isGroupExpanded: (name: string) => boolean;
|
||||||
onToggleGroup: (name: string) => void;
|
onToggleGroup: (name: string) => void;
|
||||||
|
/** Registers a group wrapper so an expanded group can be scrolled into view. */
|
||||||
|
registerGroup?: (name: string) => (node: HTMLDivElement | null) => void;
|
||||||
|
/** Notifies the reveal-on-expand mechanism when a group is toggled. */
|
||||||
|
onGroupToggle?: (name: string, willExpand: boolean) => void;
|
||||||
onFilterChange: (name: string, value: [number, number] | string[]) => void;
|
onFilterChange: (name: string, value: [number, number] | string[]) => void;
|
||||||
onRemoveFilter: (name: string) => void;
|
onRemoveFilter: (name: string) => void;
|
||||||
onDragStart: (name: string, initialValue?: [number, number]) => void;
|
onDragStart: (name: string, initialValue?: [number, number]) => void;
|
||||||
|
|
@ -76,6 +100,8 @@ export function ActiveFilterList({
|
||||||
destinationDropdownPortal,
|
destinationDropdownPortal,
|
||||||
isGroupExpanded,
|
isGroupExpanded,
|
||||||
onToggleGroup,
|
onToggleGroup,
|
||||||
|
registerGroup,
|
||||||
|
onGroupToggle,
|
||||||
onFilterChange,
|
onFilterChange,
|
||||||
onRemoveFilter,
|
onRemoveFilter,
|
||||||
onDragStart,
|
onDragStart,
|
||||||
|
|
@ -152,10 +178,11 @@ export function ActiveFilterList({
|
||||||
if (isSpecificCrimeFilterName(feature.name)) {
|
if (isSpecificCrimeFilterName(feature.name)) {
|
||||||
const specificCrimeBackendName = getSpecificCrimeFeatureName(feature.name);
|
const specificCrimeBackendName = getSpecificCrimeFeatureName(feature.name);
|
||||||
return (
|
return (
|
||||||
<SpecificCrimeFilterCard
|
<VariantFilterCard
|
||||||
key={feature.name}
|
key={feature.name}
|
||||||
|
config={SPECIFIC_CRIME_VARIANT_CONFIG}
|
||||||
features={features}
|
features={features}
|
||||||
crimeFeature={feature}
|
variantFeature={feature}
|
||||||
filters={filters}
|
filters={filters}
|
||||||
activeFeature={activeFeature}
|
activeFeature={activeFeature}
|
||||||
dragValue={dragValue}
|
dragValue={dragValue}
|
||||||
|
|
@ -177,6 +204,91 @@ export function ActiveFilterList({
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isCrimeSeverityFilterName(feature.name)) {
|
||||||
|
const crimeSeverityBackendName = getCrimeSeverityFeatureName(feature.name);
|
||||||
|
const crimeSeverityFilterName = getCrimeSeverityFilterName(feature.name);
|
||||||
|
if (!crimeSeverityFilterName) return null;
|
||||||
|
return (
|
||||||
|
<VariantFilterCard
|
||||||
|
key={feature.name}
|
||||||
|
config={getCrimeSeverityVariantConfig(crimeSeverityFilterName)}
|
||||||
|
features={features}
|
||||||
|
variantFeature={feature}
|
||||||
|
filters={filters}
|
||||||
|
activeFeature={activeFeature}
|
||||||
|
dragValue={dragValue}
|
||||||
|
pinnedFeature={pinnedFeature}
|
||||||
|
filterImpact={
|
||||||
|
crimeSeverityBackendName ? filterImpacts?.[crimeSeverityBackendName] : undefined
|
||||||
|
}
|
||||||
|
percentileScale={
|
||||||
|
crimeSeverityBackendName ? percentileScales.get(crimeSeverityBackendName) : undefined
|
||||||
|
}
|
||||||
|
onFilterChange={onFilterChange}
|
||||||
|
onDragStart={onDragStart}
|
||||||
|
onDragChange={onDragChange}
|
||||||
|
onDragEnd={onDragEnd}
|
||||||
|
onTogglePin={onTogglePin}
|
||||||
|
onShowInfo={onShowInfo}
|
||||||
|
onRemove={() => onRemoveFilter(feature.name)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isQualificationFilterName(feature.name)) {
|
||||||
|
const qualificationBackendName = getQualificationFeatureName(feature.name);
|
||||||
|
return (
|
||||||
|
<VariantFilterCard
|
||||||
|
key={feature.name}
|
||||||
|
config={QUALIFICATION_VARIANT_CONFIG}
|
||||||
|
features={features}
|
||||||
|
variantFeature={feature}
|
||||||
|
filters={filters}
|
||||||
|
activeFeature={activeFeature}
|
||||||
|
dragValue={dragValue}
|
||||||
|
pinnedFeature={pinnedFeature}
|
||||||
|
filterImpact={
|
||||||
|
qualificationBackendName ? filterImpacts?.[qualificationBackendName] : undefined
|
||||||
|
}
|
||||||
|
percentileScale={
|
||||||
|
qualificationBackendName ? percentileScales.get(qualificationBackendName) : undefined
|
||||||
|
}
|
||||||
|
onFilterChange={onFilterChange}
|
||||||
|
onDragStart={onDragStart}
|
||||||
|
onDragChange={onDragChange}
|
||||||
|
onDragEnd={onDragEnd}
|
||||||
|
onTogglePin={onTogglePin}
|
||||||
|
onShowInfo={onShowInfo}
|
||||||
|
onRemove={() => onRemoveFilter(feature.name)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isTenureFilterName(feature.name)) {
|
||||||
|
const tenureBackendName = getTenureFeatureName(feature.name);
|
||||||
|
return (
|
||||||
|
<VariantFilterCard
|
||||||
|
key={feature.name}
|
||||||
|
config={TENURE_VARIANT_CONFIG}
|
||||||
|
features={features}
|
||||||
|
variantFeature={feature}
|
||||||
|
filters={filters}
|
||||||
|
activeFeature={activeFeature}
|
||||||
|
dragValue={dragValue}
|
||||||
|
pinnedFeature={pinnedFeature}
|
||||||
|
filterImpact={tenureBackendName ? filterImpacts?.[tenureBackendName] : undefined}
|
||||||
|
percentileScale={tenureBackendName ? percentileScales.get(tenureBackendName) : undefined}
|
||||||
|
onFilterChange={onFilterChange}
|
||||||
|
onDragStart={onDragStart}
|
||||||
|
onDragChange={onDragChange}
|
||||||
|
onDragEnd={onDragEnd}
|
||||||
|
onTogglePin={onTogglePin}
|
||||||
|
onShowInfo={onShowInfo}
|
||||||
|
onRemove={() => onRemoveFilter(feature.name)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if (isElectionVoteShareFilterName(feature.name)) {
|
if (isElectionVoteShareFilterName(feature.name)) {
|
||||||
const electionVoteShareBackendName = getElectionVoteShareFeatureName(feature.name);
|
const electionVoteShareBackendName = getElectionVoteShareFeatureName(feature.name);
|
||||||
return (
|
return (
|
||||||
|
|
@ -298,11 +410,14 @@ export function ActiveFilterList({
|
||||||
if (count === 0) return null;
|
if (count === 0) return null;
|
||||||
const expanded = isGroupExpanded(group.name);
|
const expanded = isGroupExpanded(group.name);
|
||||||
return (
|
return (
|
||||||
<div key={group.name} className="shrink-0">
|
<div key={group.name} className="shrink-0" ref={registerGroup?.(group.name)}>
|
||||||
<CollapsibleGroupHeader
|
<CollapsibleGroupHeader
|
||||||
name={group.name}
|
name={group.name}
|
||||||
expanded={expanded}
|
expanded={expanded}
|
||||||
onToggle={() => onToggleGroup(group.name)}
|
onToggle={() => {
|
||||||
|
onToggleGroup(group.name);
|
||||||
|
onGroupToggle?.(group.name, !expanded);
|
||||||
|
}}
|
||||||
className="sticky top-0 z-30 px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 hover:bg-warm-200 dark:hover:bg-warm-800"
|
className="sticky top-0 z-30 px-3 py-2.5 text-sm font-bold text-navy-950 bg-warm-200 dark:bg-navy-900 dark:text-warm-100 hover:bg-warm-200 dark:hover:bg-warm-800"
|
||||||
>
|
>
|
||||||
<span className="text-xs font-medium text-warm-400 dark:text-warm-500">{count}</span>
|
<span className="text-xs font-medium text-warm-400 dark:text-warm-500">{count}</span>
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import type { RefObject } from 'react';
|
import { useCallback, type MutableRefObject, type RefObject } from 'react';
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
|
|
||||||
|
import { useRevealOnExpand } from '../../../hooks/useRevealOnExpand';
|
||||||
import type { AiFilterErrorType } from '../../../hooks/useAiFilters';
|
import type { AiFilterErrorType } from '../../../hooks/useAiFilters';
|
||||||
import type { TravelTimeEntry } from '../../../hooks/useTravelTime';
|
import type { TravelTimeEntry } from '../../../hooks/useTravelTime';
|
||||||
import type { PercentileScale } from '../../../lib/format';
|
import type { PercentileScale } from '../../../lib/format';
|
||||||
|
|
@ -107,6 +108,16 @@ export function ActiveFiltersPanel({
|
||||||
onTravelTimeToggleNoBuses,
|
onTravelTimeToggleNoBuses,
|
||||||
}: ActiveFiltersPanelProps) {
|
}: ActiveFiltersPanelProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
|
||||||
|
// Compose the parent-owned scrollRef (used to locate newly added filters) with
|
||||||
|
// the reveal-on-expand container ref so both point at the same scroll element.
|
||||||
|
const setScrollNode = useCallback(
|
||||||
|
(node: HTMLDivElement | null) => {
|
||||||
|
(scrollRef as MutableRefObject<HTMLDivElement | null>).current = node;
|
||||||
|
setContainer(node);
|
||||||
|
},
|
||||||
|
[scrollRef, setContainer]
|
||||||
|
);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
|
|
@ -156,7 +167,10 @@ export function ActiveFiltersPanel({
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
{!collapsed && (
|
{!collapsed && (
|
||||||
<div ref={scrollRef} className="md:min-h-0 md:flex-1 md:overflow-y-auto overflow-x-hidden">
|
<div
|
||||||
|
ref={setScrollNode}
|
||||||
|
className="md:min-h-0 md:flex-1 md:overflow-y-auto overflow-x-hidden"
|
||||||
|
>
|
||||||
<AiFilterInput
|
<AiFilterInput
|
||||||
loading={aiFilterLoading}
|
loading={aiFilterLoading}
|
||||||
error={aiFilterError}
|
error={aiFilterError}
|
||||||
|
|
@ -196,6 +210,8 @@ export function ActiveFiltersPanel({
|
||||||
destinationDropdownPortal={destinationDropdownPortal}
|
destinationDropdownPortal={destinationDropdownPortal}
|
||||||
isGroupExpanded={isGroupExpanded}
|
isGroupExpanded={isGroupExpanded}
|
||||||
onToggleGroup={onToggleGroup}
|
onToggleGroup={onToggleGroup}
|
||||||
|
registerGroup={registerGroup}
|
||||||
|
onGroupToggle={revealGroupOnToggle}
|
||||||
onFilterChange={onFilterChange}
|
onFilterChange={onFilterChange}
|
||||||
onRemoveFilter={onRemoveFilter}
|
onRemoveFilter={onRemoveFilter}
|
||||||
onDragStart={onDragStart}
|
onDragStart={onDragStart}
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,27 @@
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
|
|
||||||
|
import { useRevealOnExpand } from '../../../hooks/useRevealOnExpand';
|
||||||
import type { TravelTimeEntry, TransportMode } from '../../../hooks/useTravelTime';
|
import type { TravelTimeEntry, TransportMode } from '../../../hooks/useTravelTime';
|
||||||
import type { FeatureMeta } from '../../../types';
|
import type { FeatureMeta } from '../../../types';
|
||||||
import { ChevronIcon } from '../../ui/icons';
|
import { ChevronIcon } from '../../ui/icons';
|
||||||
import FeatureBrowser from '../FeatureBrowser';
|
import FeatureBrowser from '../FeatureBrowser';
|
||||||
import { SPECIFIC_CRIMES_FILTER_NAME, isSpecificCrimeFilterName } from '../../../lib/crime-filter';
|
import { SPECIFIC_CRIMES_FILTER_NAME, isSpecificCrimeFilterName } from '../../../lib/crime-filter';
|
||||||
|
import {
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES,
|
||||||
|
getCrimeSeverityFilterName,
|
||||||
|
isCrimeSeverityFilterName,
|
||||||
|
type CrimeSeverityFilterName,
|
||||||
|
} from '../../../lib/crime-severity-filter';
|
||||||
import {
|
import {
|
||||||
ELECTION_VOTE_SHARE_FILTER_NAME,
|
ELECTION_VOTE_SHARE_FILTER_NAME,
|
||||||
isElectionVoteShareFilterName,
|
isElectionVoteShareFilterName,
|
||||||
} from '../../../lib/election-filter';
|
} from '../../../lib/election-filter';
|
||||||
import { ETHNICITIES_FILTER_NAME, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
|
import { ETHNICITIES_FILTER_NAME, isEthnicityFilterName } from '../../../lib/ethnicity-filter';
|
||||||
|
import {
|
||||||
|
QUALIFICATIONS_FILTER_NAME,
|
||||||
|
isQualificationFilterName,
|
||||||
|
} from '../../../lib/qualification-filter';
|
||||||
|
import { TENURE_FILTER_NAME, isTenureFilterName } from '../../../lib/tenure-filter';
|
||||||
import { SCHOOL_FILTER_NAME, isSchoolFilterName } from '../../../lib/school-filter';
|
import { SCHOOL_FILTER_NAME, isSchoolFilterName } from '../../../lib/school-filter';
|
||||||
import {
|
import {
|
||||||
POI_DISTANCE_FILTER_NAME,
|
POI_DISTANCE_FILTER_NAME,
|
||||||
|
|
@ -27,8 +39,11 @@ interface AddFilterPanelProps {
|
||||||
pinnedFeature: string | null;
|
pinnedFeature: string | null;
|
||||||
defaultSchoolFeatureName: string | null;
|
defaultSchoolFeatureName: string | null;
|
||||||
defaultSpecificCrimeFeatureName: string | null;
|
defaultSpecificCrimeFeatureName: string | null;
|
||||||
|
defaultCrimeSeverityFeatureNames: Record<CrimeSeverityFilterName, string | null>;
|
||||||
defaultElectionVoteShareFeatureName: string | null;
|
defaultElectionVoteShareFeatureName: string | null;
|
||||||
defaultEthnicityFeatureName: string | null;
|
defaultEthnicityFeatureName: string | null;
|
||||||
|
defaultQualificationFeatureName: string | null;
|
||||||
|
defaultTenureFeatureName: string | null;
|
||||||
defaultPoiFilterFeatureNames: Record<PoiFilterName, string | null>;
|
defaultPoiFilterFeatureNames: Record<PoiFilterName, string | null>;
|
||||||
openInfoFeature?: string | null;
|
openInfoFeature?: string | null;
|
||||||
travelTimeEntries: TravelTimeEntry[];
|
travelTimeEntries: TravelTimeEntry[];
|
||||||
|
|
@ -49,8 +64,11 @@ export function AddFilterPanel({
|
||||||
pinnedFeature,
|
pinnedFeature,
|
||||||
defaultSchoolFeatureName,
|
defaultSchoolFeatureName,
|
||||||
defaultSpecificCrimeFeatureName,
|
defaultSpecificCrimeFeatureName,
|
||||||
|
defaultCrimeSeverityFeatureNames,
|
||||||
defaultElectionVoteShareFeatureName,
|
defaultElectionVoteShareFeatureName,
|
||||||
defaultEthnicityFeatureName,
|
defaultEthnicityFeatureName,
|
||||||
|
defaultQualificationFeatureName,
|
||||||
|
defaultTenureFeatureName,
|
||||||
defaultPoiFilterFeatureNames,
|
defaultPoiFilterFeatureNames,
|
||||||
openInfoFeature,
|
openInfoFeature,
|
||||||
travelTimeEntries,
|
travelTimeEntries,
|
||||||
|
|
@ -63,6 +81,7 @@ export function AddFilterPanel({
|
||||||
onUpgradeClick,
|
onUpgradeClick,
|
||||||
}: AddFilterPanelProps) {
|
}: AddFilterPanelProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const { setContainer, registerGroup, onToggle: revealGroupOnToggle } = useRevealOnExpand();
|
||||||
|
|
||||||
const browserPinnedFeature =
|
const browserPinnedFeature =
|
||||||
pinnedFeature && isSchoolFilterName(pinnedFeature)
|
pinnedFeature && isSchoolFilterName(pinnedFeature)
|
||||||
|
|
@ -73,9 +92,15 @@ export function AddFilterPanel({
|
||||||
? ELECTION_VOTE_SHARE_FILTER_NAME
|
? ELECTION_VOTE_SHARE_FILTER_NAME
|
||||||
: pinnedFeature && isEthnicityFilterName(pinnedFeature)
|
: pinnedFeature && isEthnicityFilterName(pinnedFeature)
|
||||||
? ETHNICITIES_FILTER_NAME
|
? ETHNICITIES_FILTER_NAME
|
||||||
: pinnedFeature && isPoiDistanceFilterName(pinnedFeature)
|
: pinnedFeature && isQualificationFilterName(pinnedFeature)
|
||||||
? (getPoiFilterName(pinnedFeature) ?? POI_DISTANCE_FILTER_NAME)
|
? QUALIFICATIONS_FILTER_NAME
|
||||||
: pinnedFeature;
|
: pinnedFeature && isTenureFilterName(pinnedFeature)
|
||||||
|
? TENURE_FILTER_NAME
|
||||||
|
: pinnedFeature && isPoiDistanceFilterName(pinnedFeature)
|
||||||
|
? (getPoiFilterName(pinnedFeature) ?? POI_DISTANCE_FILTER_NAME)
|
||||||
|
: pinnedFeature && isCrimeSeverityFilterName(pinnedFeature)
|
||||||
|
? (getCrimeSeverityFilterName(pinnedFeature) ?? pinnedFeature)
|
||||||
|
: pinnedFeature;
|
||||||
|
|
||||||
const handleTogglePin = (name: string) => {
|
const handleTogglePin = (name: string) => {
|
||||||
if (name === SCHOOL_FILTER_NAME) {
|
if (name === SCHOOL_FILTER_NAME) {
|
||||||
|
|
@ -94,6 +119,20 @@ export function AddFilterPanel({
|
||||||
if (defaultEthnicityFeatureName) onTogglePin(defaultEthnicityFeatureName);
|
if (defaultEthnicityFeatureName) onTogglePin(defaultEthnicityFeatureName);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (name === QUALIFICATIONS_FILTER_NAME) {
|
||||||
|
if (defaultQualificationFeatureName) onTogglePin(defaultQualificationFeatureName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (name === TENURE_FILTER_NAME) {
|
||||||
|
if (defaultTenureFeatureName) onTogglePin(defaultTenureFeatureName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
|
||||||
|
const defaultSeverityFeatureName =
|
||||||
|
defaultCrimeSeverityFeatureNames[name as CrimeSeverityFilterName];
|
||||||
|
if (defaultSeverityFeatureName) onTogglePin(defaultSeverityFeatureName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
||||||
const defaultPoiFeatureName = defaultPoiFilterFeatureNames[name as PoiFilterName];
|
const defaultPoiFeatureName = defaultPoiFilterFeatureNames[name as PoiFilterName];
|
||||||
if (defaultPoiFeatureName) onTogglePin(defaultPoiFeatureName);
|
if (defaultPoiFeatureName) onTogglePin(defaultPoiFeatureName);
|
||||||
|
|
@ -123,7 +162,7 @@ export function AddFilterPanel({
|
||||||
{(!collapsed || !isLicensed) && (
|
{(!collapsed || !isLicensed) && (
|
||||||
<div className="flex min-h-0 flex-1 flex-col">
|
<div className="flex min-h-0 flex-1 flex-col">
|
||||||
{!collapsed && (
|
{!collapsed && (
|
||||||
<div className="min-h-0 flex-1 overflow-y-auto">
|
<div ref={setContainer} className="min-h-0 flex-1 overflow-y-auto">
|
||||||
<FeatureBrowser
|
<FeatureBrowser
|
||||||
availableFeatures={availableFeatures}
|
availableFeatures={availableFeatures}
|
||||||
allFeatures={allFeatures}
|
allFeatures={allFeatures}
|
||||||
|
|
@ -135,6 +174,8 @@ export function AddFilterPanel({
|
||||||
onClearOpenInfoFeature={onClearOpenInfoFeature}
|
onClearOpenInfoFeature={onClearOpenInfoFeature}
|
||||||
travelTimeEntries={travelTimeEntries}
|
travelTimeEntries={travelTimeEntries}
|
||||||
onAddTravelTimeEntry={onAddTravelTimeEntry}
|
onAddTravelTimeEntry={onAddTravelTimeEntry}
|
||||||
|
registerGroup={registerGroup}
|
||||||
|
onGroupToggle={revealGroupOnToggle}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
|
||||||
|
|
@ -5,23 +5,22 @@ import { ChevronIcon } from '../../ui/icons';
|
||||||
import { FeatureActions } from '../../ui/FeatureIcons';
|
import { FeatureActions } from '../../ui/FeatureIcons';
|
||||||
import { FeatureLabel } from '../../ui/FeatureLabel';
|
import { FeatureLabel } from '../../ui/FeatureLabel';
|
||||||
import type { FeatureFilters, FeatureMeta } from '../../../types';
|
import type { FeatureFilters, FeatureMeta } from '../../../types';
|
||||||
|
import type { VariantFilterConfig } from '../../../lib/variant-filter';
|
||||||
import { formatNumber, type PercentileScale } from '../../../lib/format';
|
import { formatNumber, type PercentileScale } from '../../../lib/format';
|
||||||
import { getFeatureIcon } from '../../../lib/feature-icons';
|
import { getFeatureIcon } from '../../../lib/feature-icons';
|
||||||
import { getGroupIcon } from '../../../lib/group-icons';
|
import { getGroupIcon } from '../../../lib/group-icons';
|
||||||
import {
|
|
||||||
SPECIFIC_CRIMES_FILTER_NAME,
|
|
||||||
SPECIFIC_CRIME_FEATURE_NAMES,
|
|
||||||
clampSpecificCrimeRange,
|
|
||||||
getDefaultSpecificCrimeFeatureName,
|
|
||||||
getSpecificCrimeFeatureName,
|
|
||||||
getSpecificCrimeFilterMeta,
|
|
||||||
replaceSpecificCrimeFilterKeySelection,
|
|
||||||
} from '../../../lib/crime-filter';
|
|
||||||
import { SliderLabels } from './SliderLabels';
|
import { SliderLabels } from './SliderLabels';
|
||||||
|
|
||||||
export function SpecificCrimeFilterCard({
|
/**
|
||||||
|
* One card for a "pick a variant, filter its range" aggregate filter
|
||||||
|
* (specific crimes, qualifications, …). Behaviour is variant-agnostic: the
|
||||||
|
* `config` supplies the dropdown options, labels and helper functions so the
|
||||||
|
* same component backs every such filter (see [`VariantFilterConfig`]).
|
||||||
|
*/
|
||||||
|
export function VariantFilterCard({
|
||||||
|
config,
|
||||||
features,
|
features,
|
||||||
crimeFeature,
|
variantFeature,
|
||||||
filters,
|
filters,
|
||||||
activeFeature,
|
activeFeature,
|
||||||
dragValue,
|
dragValue,
|
||||||
|
|
@ -36,8 +35,9 @@ export function SpecificCrimeFilterCard({
|
||||||
onShowInfo,
|
onShowInfo,
|
||||||
onRemove,
|
onRemove,
|
||||||
}: {
|
}: {
|
||||||
|
config: VariantFilterConfig;
|
||||||
features: FeatureMeta[];
|
features: FeatureMeta[];
|
||||||
crimeFeature: FeatureMeta;
|
variantFeature: FeatureMeta;
|
||||||
filters: FeatureFilters;
|
filters: FeatureFilters;
|
||||||
activeFeature: string | null;
|
activeFeature: string | null;
|
||||||
dragValue: [number, number] | null;
|
dragValue: [number, number] | null;
|
||||||
|
|
@ -53,27 +53,62 @@ export function SpecificCrimeFilterCard({
|
||||||
onRemove: () => void;
|
onRemove: () => void;
|
||||||
}) {
|
}) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const specificCrimeMeta = getSpecificCrimeFilterMeta(features);
|
const variantMeta = config.getFilterMeta(features);
|
||||||
const crimeOptions = SPECIFIC_CRIME_FEATURE_NAMES.map((name) =>
|
|
||||||
features.find((feature) => feature.name === name)
|
|
||||||
).filter((feature): feature is FeatureMeta => Boolean(feature));
|
|
||||||
const selectedFeatureName =
|
const selectedFeatureName =
|
||||||
getSpecificCrimeFeatureName(crimeFeature.name) ?? getDefaultSpecificCrimeFeatureName(features);
|
config.getFeatureName(variantFeature.name) ?? config.getDefaultFeatureName(features);
|
||||||
const selectedFeature = selectedFeatureName
|
const selectedFeature = selectedFeatureName
|
||||||
? features.find((feature) => feature.name === selectedFeatureName)
|
? features.find((feature) => feature.name === selectedFeatureName)
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
if (!selectedFeature || crimeOptions.length === 0 || !selectedFeatureName) return null;
|
// Optional secondary axis: the same variant measured over a different window
|
||||||
|
// (e.g. 7- vs 2-year crime rates). The current window comes from the selected
|
||||||
|
// feature so the dropdown can re-point each option into that window.
|
||||||
|
const windowConfig = config.window;
|
||||||
|
const currentWindow =
|
||||||
|
(windowConfig && selectedFeatureName ? windowConfig.getWindow(selectedFeatureName) : null) ??
|
||||||
|
windowConfig?.options[0]?.id ??
|
||||||
|
null;
|
||||||
|
|
||||||
const isActive = activeFeature === crimeFeature.name;
|
const variantOptions = config.featureNames
|
||||||
const isPinned = pinnedFeature === crimeFeature.name;
|
.map((canonicalName) => {
|
||||||
|
const optionName =
|
||||||
|
windowConfig && currentWindow
|
||||||
|
? windowConfig.withWindow(canonicalName, currentWindow)
|
||||||
|
: canonicalName;
|
||||||
|
const feature = features.find((f) => f.name === optionName);
|
||||||
|
if (!feature) return null;
|
||||||
|
return {
|
||||||
|
value: optionName,
|
||||||
|
label: ts(config.getOptionLabelSource ? config.getOptionLabelSource(optionName) : optionName),
|
||||||
|
feature,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((option): option is { value: string; label: string; feature: FeatureMeta } =>
|
||||||
|
Boolean(option)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Only offer windows that actually exist for the selected variant, so a
|
||||||
|
// missing backend column can't strand the card on an empty selection.
|
||||||
|
const windowOptions =
|
||||||
|
windowConfig && selectedFeatureName
|
||||||
|
? windowConfig.options.filter((option) =>
|
||||||
|
features.some(
|
||||||
|
(f) => f.name === windowConfig.withWindow(selectedFeatureName, option.id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
if (!selectedFeature || variantOptions.length === 0 || !selectedFeatureName) return null;
|
||||||
|
|
||||||
|
const isActive = activeFeature === variantFeature.name;
|
||||||
|
const isPinned = pinnedFeature === variantFeature.name;
|
||||||
const hist = selectedFeature.histogram;
|
const hist = selectedFeature.histogram;
|
||||||
const dataMin = hist?.min ?? selectedFeature.min ?? 0;
|
const dataMin = hist?.min ?? selectedFeature.min ?? 0;
|
||||||
const dataMax = hist?.max ?? selectedFeature.max ?? 100;
|
const dataMax = hist?.max ?? selectedFeature.max ?? 100;
|
||||||
const displayValue =
|
const displayValue =
|
||||||
isActive && dragValue
|
isActive && dragValue
|
||||||
? dragValue
|
? dragValue
|
||||||
: (filters[crimeFeature.name] as [number, number]) || [dataMin, dataMax];
|
: (filters[variantFeature.name] as [number, number]) || [dataMin, dataMax];
|
||||||
const scale = percentileScale;
|
const scale = percentileScale;
|
||||||
const clampMin = displayValue[0] <= dataMin;
|
const clampMin = displayValue[0] <= dataMin;
|
||||||
const clampMax = displayValue[1] >= dataMax;
|
const clampMax = displayValue[1] >= dataMax;
|
||||||
|
|
@ -89,14 +124,14 @@ export function SpecificCrimeFilterCard({
|
||||||
clampMax ? (selectedFeature.max ?? dataMax) : displayValue[1],
|
clampMax ? (selectedFeature.max ?? dataMax) : displayValue[1],
|
||||||
];
|
];
|
||||||
|
|
||||||
const replaceCrimeFeature = (nextFeatureName: string) => {
|
const replaceVariantFeature = (nextFeatureName: string) => {
|
||||||
const nextName = replaceSpecificCrimeFilterKeySelection(crimeFeature.name, nextFeatureName);
|
const nextName = config.replaceFilterKeySelection(variantFeature.name, nextFeatureName);
|
||||||
if (nextName === crimeFeature.name) return;
|
if (nextName === variantFeature.name) return;
|
||||||
|
|
||||||
const nextFeature = features.find((feature) => feature.name === nextFeatureName);
|
const nextFeature = features.find((feature) => feature.name === nextFeatureName);
|
||||||
const nextDataMin = nextFeature?.histogram?.min ?? nextFeature?.min ?? 0;
|
const nextDataMin = nextFeature?.histogram?.min ?? nextFeature?.min ?? 0;
|
||||||
const nextDataMax = nextFeature?.histogram?.max ?? nextFeature?.max ?? Math.max(1, dataMax);
|
const nextDataMax = nextFeature?.histogram?.max ?? nextFeature?.max ?? Math.max(1, dataMax);
|
||||||
const nextRange = clampSpecificCrimeRange(
|
const nextRange = config.clampRange(
|
||||||
[
|
[
|
||||||
displayValue[0] <= dataMin ? nextDataMin : displayValue[0],
|
displayValue[0] <= dataMin ? nextDataMin : displayValue[0],
|
||||||
displayValue[1] >= dataMax ? nextDataMax : displayValue[1],
|
displayValue[1] >= dataMax ? nextDataMax : displayValue[1],
|
||||||
|
|
@ -108,6 +143,11 @@ export function SpecificCrimeFilterCard({
|
||||||
if (isPinned) onTogglePin(nextName);
|
if (isPinned) onTogglePin(nextName);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const switchWindow = (windowId: string) => {
|
||||||
|
if (!windowConfig || windowId === currentWindow) return;
|
||||||
|
replaceVariantFeature(windowConfig.withWindow(selectedFeatureName, windowId));
|
||||||
|
};
|
||||||
|
|
||||||
const mobileIconClass = 'w-4 h-4 text-teal-600 dark:text-teal-400 shrink-0';
|
const mobileIconClass = 'w-4 h-4 text-teal-600 dark:text-teal-400 shrink-0';
|
||||||
const mobileIcon =
|
const mobileIcon =
|
||||||
getFeatureIcon(selectedFeature.name, mobileIconClass) ||
|
getFeatureIcon(selectedFeature.name, mobileIconClass) ||
|
||||||
|
|
@ -118,7 +158,7 @@ export function SpecificCrimeFilterCard({
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
data-filter-name={SPECIFIC_CRIMES_FILTER_NAME}
|
data-filter-name={config.filterName}
|
||||||
className={`space-y-1.5 px-2 py-1.5 rounded ${
|
className={`space-y-1.5 px-2 py-1.5 rounded ${
|
||||||
isActive
|
isActive
|
||||||
? 'ring-2 ring-teal-400 bg-teal-50 dark:bg-teal-900/30'
|
? 'ring-2 ring-teal-400 bg-teal-50 dark:bg-teal-900/30'
|
||||||
|
|
@ -129,7 +169,7 @@ export function SpecificCrimeFilterCard({
|
||||||
>
|
>
|
||||||
<div className="relative z-10 flex items-center justify-between gap-1">
|
<div className="relative z-10 flex items-center justify-between gap-1">
|
||||||
<FeatureLabel
|
<FeatureLabel
|
||||||
feature={specificCrimeMeta}
|
feature={variantMeta}
|
||||||
size="sm"
|
size="sm"
|
||||||
className="min-w-0 shrink"
|
className="min-w-0 shrink"
|
||||||
hideIconOnMobile
|
hideIconOnMobile
|
||||||
|
|
@ -137,7 +177,7 @@ export function SpecificCrimeFilterCard({
|
||||||
/>
|
/>
|
||||||
<FeatureActions
|
<FeatureActions
|
||||||
feature={selectedFeature}
|
feature={selectedFeature}
|
||||||
actionName={crimeFeature.name}
|
actionName={variantFeature.name}
|
||||||
isPinned={isPinned}
|
isPinned={isPinned}
|
||||||
isPreviewing={isActive}
|
isPreviewing={isActive}
|
||||||
onTogglePin={onTogglePin}
|
onTogglePin={onTogglePin}
|
||||||
|
|
@ -146,28 +186,65 @@ export function SpecificCrimeFilterCard({
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div>
|
{/* A single-variant filter (e.g. Serious/Minor crime) has nothing to pick,
|
||||||
<label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500">
|
so the dropdown is hidden and only the window toggle + slider remain. */}
|
||||||
{t('filters.crimeType')}
|
{variantOptions.length > 1 && (
|
||||||
</label>
|
<div>
|
||||||
<div className="relative">
|
<label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500">
|
||||||
<select
|
{t(config.dropdownLabelKey)}
|
||||||
value={selectedFeatureName}
|
</label>
|
||||||
onChange={(e) => replaceCrimeFeature(e.target.value)}
|
<div className="relative">
|
||||||
className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50"
|
<select
|
||||||
>
|
value={selectedFeatureName}
|
||||||
{crimeOptions.map((option) => (
|
onChange={(e) => replaceVariantFeature(e.target.value)}
|
||||||
<option key={option.name} value={option.name}>
|
className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50"
|
||||||
{ts(option.name)}
|
>
|
||||||
</option>
|
{variantOptions.map((option) => (
|
||||||
))}
|
<option key={option.value} value={option.value}>
|
||||||
</select>
|
{option.label}
|
||||||
<ChevronIcon
|
</option>
|
||||||
direction="down"
|
))}
|
||||||
className="pointer-events-none absolute right-2 top-1/2 h-4 w-4 -translate-y-1/2 text-warm-400 dark:text-warm-500"
|
</select>
|
||||||
/>
|
<ChevronIcon
|
||||||
|
direction="down"
|
||||||
|
className="pointer-events-none absolute right-2 top-1/2 h-4 w-4 -translate-y-1/2 text-warm-400 dark:text-warm-500"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
)}
|
||||||
|
|
||||||
|
{windowConfig && currentWindow && windowOptions.length > 1 && (
|
||||||
|
<div>
|
||||||
|
{windowConfig.labelKey && (
|
||||||
|
<label className="mb-1 block text-[10px] font-medium uppercase text-warm-400 dark:text-warm-500">
|
||||||
|
{t(windowConfig.labelKey)}
|
||||||
|
</label>
|
||||||
|
)}
|
||||||
|
<div
|
||||||
|
role="group"
|
||||||
|
className="inline-flex rounded-md border border-warm-200 bg-warm-50 p-0.5 dark:border-warm-700 dark:bg-navy-900"
|
||||||
|
>
|
||||||
|
{windowOptions.map((option) => {
|
||||||
|
const active = option.id === currentWindow;
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={option.id}
|
||||||
|
type="button"
|
||||||
|
aria-pressed={active}
|
||||||
|
onClick={() => switchWindow(option.id)}
|
||||||
|
className={`rounded px-2.5 py-1 text-xs font-medium transition-colors ${
|
||||||
|
active
|
||||||
|
? 'bg-teal-600 text-white shadow-sm'
|
||||||
|
: 'text-warm-500 hover:text-navy-950 dark:text-warm-400 dark:hover:text-warm-100'
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{t(option.labelKey)}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="flex items-start gap-1.5 md:block">
|
<div className="flex items-start gap-1.5 md:block">
|
||||||
{mobileIcon && <div className="shrink-0 pt-0.5 md:hidden">{mobileIcon}</div>}
|
{mobileIcon && <div className="shrink-0 pt-0.5 md:hidden">{mobileIcon}</div>}
|
||||||
|
|
@ -198,7 +275,7 @@ export function SpecificCrimeFilterCard({
|
||||||
max >= (selectedFeature.max ?? dataMax) ? dataMax : max,
|
max >= (selectedFeature.max ?? dataMax) ? dataMax : max,
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
onPointerDown={() => onDragStart(crimeFeature.name, displayValue)}
|
onPointerDown={() => onDragStart(variantFeature.name, displayValue)}
|
||||||
onPointerUp={() => onDragEnd()}
|
onPointerUp={() => onDragEnd()}
|
||||||
/>
|
/>
|
||||||
<SliderLabels
|
<SliderLabels
|
||||||
|
|
@ -211,7 +288,7 @@ export function SpecificCrimeFilterCard({
|
||||||
raw={selectedFeature.raw}
|
raw={selectedFeature.raw}
|
||||||
feature={selectedFeature}
|
feature={selectedFeature}
|
||||||
onValueChange={(v) =>
|
onValueChange={(v) =>
|
||||||
onFilterChange(crimeFeature.name, clampSpecificCrimeRange(v, selectedFeature))
|
onFilterChange(variantFeature.name, config.clampRange(v, selectedFeature))
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
{filterImpact != null && filterImpact > 0 && (
|
{filterImpact != null && filterImpact > 0 && (
|
||||||
|
|
@ -236,8 +236,8 @@ export function DesktopMapPage({
|
||||||
onClick={onToggleActualListings}
|
onClick={onToggleActualListings}
|
||||||
aria-pressed={actualListingsEnabled}
|
aria-pressed={actualListingsEnabled}
|
||||||
aria-busy={actualListingsLoading}
|
aria-busy={actualListingsLoading}
|
||||||
aria-label={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
|
aria-label={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
|
||||||
title={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
|
title={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
|
||||||
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
|
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
|
||||||
>
|
>
|
||||||
{actualListingsLoading ? (
|
{actualListingsLoading ? (
|
||||||
|
|
@ -246,16 +246,17 @@ export function DesktopMapPage({
|
||||||
<HouseIcon className="h-5 w-5" />
|
<HouseIcon className="h-5 w-5" />
|
||||||
)}
|
)}
|
||||||
<span className="text-sm font-medium">
|
<span className="text-sm font-medium">
|
||||||
Listings{actualListingsEnabled ? ` (${actualListings.length})` : ''}
|
{t('map.actualListings.label')}{actualListingsEnabled ? ` (${actualListings.length})` : ''}
|
||||||
</span>
|
</span>
|
||||||
</button>
|
</button>
|
||||||
)}
|
)}
|
||||||
<button
|
<button
|
||||||
|
data-tutorial="overlays-button"
|
||||||
onClick={onToggleOverlayPane}
|
onClick={onToggleOverlayPane}
|
||||||
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
|
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
|
||||||
>
|
>
|
||||||
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
|
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
|
||||||
<span className="text-sm font-medium">Overlays</span>
|
<span className="text-sm font-medium">{t('overlays.heading')}</span>
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
data-tutorial="poi-button"
|
data-tutorial="poi-button"
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { Suspense, type MutableRefObject, type ReactNode } from 'react';
|
import { Suspense, type MutableRefObject, type ReactNode } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
|
||||||
import type {
|
import type {
|
||||||
ActualListing,
|
ActualListing,
|
||||||
|
|
@ -133,6 +134,7 @@ export function MobileMapPage({
|
||||||
upgradeModal,
|
upgradeModal,
|
||||||
editingBar,
|
editingBar,
|
||||||
}: MobileMapPageProps) {
|
}: MobileMapPageProps) {
|
||||||
|
const { t } = useTranslation();
|
||||||
const floatingPaneAvailableHeight = `max(12rem, calc(100dvh - ${Math.ceil(
|
const floatingPaneAvailableHeight = `max(12rem, calc(100dvh - ${Math.ceil(
|
||||||
bottomScreenInset
|
bottomScreenInset
|
||||||
)}px - 7rem))`;
|
)}px - 7rem))`;
|
||||||
|
|
@ -204,8 +206,16 @@ export function MobileMapPage({
|
||||||
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
|
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
|
||||||
aria-pressed={actualListingsEnabled}
|
aria-pressed={actualListingsEnabled}
|
||||||
aria-busy={actualListingsLoading}
|
aria-busy={actualListingsLoading}
|
||||||
aria-label={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
|
aria-label={
|
||||||
title={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
|
actualListingsEnabled
|
||||||
|
? t('map.actualListings.hide')
|
||||||
|
: t('map.actualListings.show')
|
||||||
|
}
|
||||||
|
title={
|
||||||
|
actualListingsEnabled
|
||||||
|
? t('map.actualListings.hide')
|
||||||
|
: t('map.actualListings.show')
|
||||||
|
}
|
||||||
>
|
>
|
||||||
{actualListingsLoading ? (
|
{actualListingsLoading ? (
|
||||||
<SpinnerIcon className="h-5 w-5 animate-spin" />
|
<SpinnerIcon className="h-5 w-5 animate-spin" />
|
||||||
|
|
@ -217,7 +227,7 @@ export function MobileMapPage({
|
||||||
<button
|
<button
|
||||||
onClick={onToggleOverlayPane}
|
onClick={onToggleOverlayPane}
|
||||||
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
|
className={`rounded-lg bg-white p-2 shadow-lg dark:bg-warm-800 ${overlayPaneOpen ? 'text-teal-600 dark:text-teal-400' : 'text-warm-500 hover:text-teal-600 dark:text-warm-400 dark:hover:text-teal-400'}`}
|
||||||
aria-label="Overlays"
|
aria-label={t('overlays.heading')}
|
||||||
>
|
>
|
||||||
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
|
<EyeIcon className="h-5 w-5" filled={overlayPaneOpen} />
|
||||||
</button>
|
</button>
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,11 @@ import type { HexagonLocation } from '../../../lib/external-search';
|
||||||
import type { useMapData } from '../../../hooks/useMapData';
|
import type { useMapData } from '../../../hooks/useMapData';
|
||||||
import { resolveTransitVariant, type TravelTimeEntry } from '../../../hooks/useTravelTime';
|
import { resolveTransitVariant, type TravelTimeEntry } from '../../../hooks/useTravelTime';
|
||||||
import { getSpecificCrimeFeatureName } from '../../../lib/crime-filter';
|
import { getSpecificCrimeFeatureName } from '../../../lib/crime-filter';
|
||||||
|
import { getCrimeSeverityFeatureName } from '../../../lib/crime-severity-filter';
|
||||||
import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
|
import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
|
||||||
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
|
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
|
||||||
|
import { getQualificationFeatureName } from '../../../lib/qualification-filter';
|
||||||
|
import { getTenureFeatureName } from '../../../lib/tenure-filter';
|
||||||
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
|
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
|
||||||
import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
|
import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
|
||||||
|
|
||||||
|
|
@ -23,8 +26,11 @@ export function getMapPageBackendFeatureName(featureName: string): string {
|
||||||
return (
|
return (
|
||||||
getSchoolBackendFeatureName(featureName) ??
|
getSchoolBackendFeatureName(featureName) ??
|
||||||
getSpecificCrimeFeatureName(featureName) ??
|
getSpecificCrimeFeatureName(featureName) ??
|
||||||
|
getCrimeSeverityFeatureName(featureName) ??
|
||||||
getElectionVoteShareFeatureName(featureName) ??
|
getElectionVoteShareFeatureName(featureName) ??
|
||||||
getEthnicityFeatureName(featureName) ??
|
getEthnicityFeatureName(featureName) ??
|
||||||
|
getQualificationFeatureName(featureName) ??
|
||||||
|
getTenureFeatureName(featureName) ??
|
||||||
getPoiDistanceFeatureName(featureName) ??
|
getPoiDistanceFeatureName(featureName) ??
|
||||||
featureName
|
featureName
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,20 @@
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
|
||||||
interface IndeterminateProgressBarProps {
|
interface IndeterminateProgressBarProps {
|
||||||
show: boolean;
|
show: boolean;
|
||||||
className?: string;
|
className?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function IndeterminateProgressBar({ show, className = '' }: IndeterminateProgressBarProps) {
|
export function IndeterminateProgressBar({ show, className = '' }: IndeterminateProgressBarProps) {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
|
||||||
if (!show) return null;
|
if (!show) return null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
role="progressbar"
|
role="progressbar"
|
||||||
aria-busy="true"
|
aria-busy="true"
|
||||||
aria-valuetext="loading"
|
aria-valuetext={t('common.loading')}
|
||||||
className={`pointer-events-none absolute top-0 left-0 right-0 z-30 h-0.5 overflow-hidden bg-teal-500/10 dark:bg-teal-400/10 animate-fade-in ${className}`}
|
className={`pointer-events-none absolute top-0 left-0 right-0 z-30 h-0.5 overflow-hidden bg-teal-500/10 dark:bg-teal-400/10 animate-fade-in ${className}`}
|
||||||
>
|
>
|
||||||
<div className="h-full w-1/4 bg-teal-500 dark:bg-teal-400 animate-indeterminate-progress" />
|
<div className="h-full w-1/4 bg-teal-500 dark:bg-teal-400 animate-indeterminate-progress" />
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { useState, useCallback, useRef } from 'react';
|
import { useState, useCallback, useRef } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import type { FeatureFilters } from '../types';
|
import type { FeatureFilters } from '../types';
|
||||||
import { apiUrl, authHeaders, logNonAbortError } from '../lib/api';
|
import { apiUrl, authHeaders, logNonAbortError } from '../lib/api';
|
||||||
|
|
||||||
|
|
@ -88,6 +89,7 @@ function buildSummary(
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useAiFilters(): UseAiFiltersResult {
|
export function useAiFilters(): UseAiFiltersResult {
|
||||||
|
const { t } = useTranslation();
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [errorType, setErrorType] = useState<AiFilterErrorType | null>(null);
|
const [errorType, setErrorType] = useState<AiFilterErrorType | null>(null);
|
||||||
|
|
@ -170,14 +172,14 @@ export function useAiFilters(): UseAiFiltersResult {
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (controller.signal.aborted) return null;
|
if (controller.signal.aborted) return null;
|
||||||
logNonAbortError('ai-filters', err);
|
logNonAbortError('ai-filters', err);
|
||||||
const message = err instanceof Error ? err.message : 'Failed to generate filters';
|
const message = err instanceof Error ? err.message : t('aiFilter.generateFailed');
|
||||||
setErrorType('error');
|
setErrorType('error');
|
||||||
setError(message);
|
setError(message);
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[]
|
[t]
|
||||||
);
|
);
|
||||||
|
|
||||||
return { fetchAiFilters, loading, error, errorType, notes, summary };
|
return { fetchAiFilters, loading, error, errorType, notes, summary };
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { useState, useEffect, useCallback } from 'react';
|
import { useState, useEffect, useCallback } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import pb from '../lib/pocketbase';
|
import pb from '../lib/pocketbase';
|
||||||
import { trackEvent } from '../lib/analytics';
|
import { trackEvent } from '../lib/analytics';
|
||||||
|
|
||||||
|
|
@ -31,6 +32,7 @@ function authRefreshInvalidated(error: unknown): boolean {
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useAuth() {
|
export function useAuth() {
|
||||||
|
const { t } = useTranslation();
|
||||||
const [user, setUser] = useState<AuthUser | null>(() => {
|
const [user, setUser] = useState<AuthUser | null>(() => {
|
||||||
if (pb.authStore.isValid && pb.authStore.record) {
|
if (pb.authStore.isValid && pb.authStore.record) {
|
||||||
return recordToUser(pb.authStore.record);
|
return recordToUser(pb.authStore.record);
|
||||||
|
|
@ -60,13 +62,13 @@ export function useAuth() {
|
||||||
setUser(recordToUser(result.record));
|
setUser(recordToUser(result.record));
|
||||||
trackEvent('Login', { method: 'email' });
|
trackEvent('Login', { method: 'email' });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Login failed';
|
const msg = err instanceof Error ? err.message : t('auth.loginFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const register = useCallback(async (email: string, password: string) => {
|
const register = useCallback(async (email: string, password: string) => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
|
|
@ -83,13 +85,13 @@ export function useAuth() {
|
||||||
setUser(recordToUser(result.record));
|
setUser(recordToUser(result.record));
|
||||||
trackEvent('Register');
|
trackEvent('Register');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Registration failed';
|
const msg = err instanceof Error ? err.message : t('auth.registrationFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const loginWithOAuth = useCallback(async (provider: string) => {
|
const loginWithOAuth = useCallback(async (provider: string) => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
|
|
@ -102,13 +104,13 @@ export function useAuth() {
|
||||||
setUser(recordToUser(result.record));
|
setUser(recordToUser(result.record));
|
||||||
trackEvent('Login', { method: provider });
|
trackEvent('Login', { method: provider });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'OAuth login failed';
|
const msg = err instanceof Error ? err.message : t('auth.oauthFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const logout = useCallback(() => {
|
const logout = useCallback(() => {
|
||||||
trackEvent('Logout');
|
trackEvent('Logout');
|
||||||
|
|
@ -122,13 +124,13 @@ export function useAuth() {
|
||||||
try {
|
try {
|
||||||
await pb.collection('users').requestPasswordReset(email);
|
await pb.collection('users').requestPasswordReset(email);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Password reset request failed';
|
const msg = err instanceof Error ? err.message : t('auth.passwordResetFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const refreshAuth = useCallback(async () => {
|
const refreshAuth = useCallback(async () => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
|
|
|
||||||
98
frontend/src/hooks/useClickedListings.test.ts
Normal file
98
frontend/src/hooks/useClickedListings.test.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
import { act, renderHook, waitFor } from '@testing-library/react';
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const mocks = vi.hoisted(() => ({
|
||||||
|
create: vi.fn(),
|
||||||
|
getFullList: vi.fn(),
|
||||||
|
authStore: {
|
||||||
|
isValid: true,
|
||||||
|
record: { id: 'user-1' } as { id: string } | null,
|
||||||
|
onChange: () => () => {},
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('../lib/pocketbase', () => ({
|
||||||
|
default: {
|
||||||
|
authStore: mocks.authStore,
|
||||||
|
collection: () => ({ getFullList: mocks.getFullList, create: mocks.create }),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { useClickedListings } from './useClickedListings';
|
||||||
|
|
||||||
|
describe('useClickedListings', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
mocks.create.mockReset().mockResolvedValue({});
|
||||||
|
mocks.getFullList.mockReset().mockResolvedValue([]);
|
||||||
|
mocks.authStore.isValid = true;
|
||||||
|
mocks.authStore.record = { id: 'user-1' };
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('hydrates the visited set from PocketBase for the signed-in user', async () => {
|
||||||
|
mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
|
||||||
|
await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records a click optimistically and persists it', async () => {
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
|
||||||
|
|
||||||
|
act(() => result.current.markClicked('https://example.com/b'));
|
||||||
|
|
||||||
|
// Visible immediately — no need to await the network round-trip.
|
||||||
|
expect(result.current.clickedUrls.has('https://example.com/b')).toBe(true);
|
||||||
|
expect(mocks.create).toHaveBeenCalledWith({
|
||||||
|
user: 'user-1',
|
||||||
|
url: 'https://example.com/b',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('produces a new Set instance for a new url so deck.gl re-colours', async () => {
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
|
||||||
|
|
||||||
|
const before = result.current.clickedUrls;
|
||||||
|
act(() => result.current.markClicked('https://example.com/c'));
|
||||||
|
expect(result.current.clickedUrls).not.toBe(before);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not re-persist an already visited url', async () => {
|
||||||
|
mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
|
||||||
|
|
||||||
|
const before = result.current.clickedUrls;
|
||||||
|
act(() => result.current.markClicked('https://example.com/a'));
|
||||||
|
|
||||||
|
expect(mocks.create).not.toHaveBeenCalled();
|
||||||
|
expect(result.current.clickedUrls).toBe(before); // identity stable → no needless recompute
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores empty urls', async () => {
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
|
||||||
|
|
||||||
|
act(() => result.current.markClicked(''));
|
||||||
|
act(() => result.current.markClicked(undefined));
|
||||||
|
|
||||||
|
expect(mocks.create).not.toHaveBeenCalled();
|
||||||
|
expect(result.current.clickedUrls.size).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('recolours for anonymous users in-memory but does not persist', async () => {
|
||||||
|
mocks.authStore.isValid = false;
|
||||||
|
mocks.authStore.record = null;
|
||||||
|
const { result } = renderHook(() => useClickedListings());
|
||||||
|
|
||||||
|
act(() => result.current.markClicked('https://example.com/d'));
|
||||||
|
|
||||||
|
expect(result.current.clickedUrls.has('https://example.com/d')).toBe(true);
|
||||||
|
expect(mocks.create).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
82
frontend/src/hooks/useClickedListings.ts
Normal file
82
frontend/src/hooks/useClickedListings.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
import pb from '../lib/pocketbase';
|
||||||
|
|
||||||
|
/** Per-user record of opened listings, stored in PocketBase so visited listings can be drawn
|
||||||
|
* in a distinct colour on the map and persist across devices. One row per (user, url). */
|
||||||
|
const CLICKED_LISTINGS_COLLECTION = 'clicked_listings';
|
||||||
|
|
||||||
|
function currentUserId(): string | null {
|
||||||
|
return pb.authStore.isValid && pb.authStore.record ? pb.authStore.record.id : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tracks which listings the signed-in user has opened.
|
||||||
|
*
|
||||||
|
* Reads the user from the shared PocketBase authStore so it stays self-contained — no need to
|
||||||
|
* thread a user id down through the map layer tree. Returns a Set for O(1) membership checks in
|
||||||
|
* the map's colour accessor; a fresh Set instance is produced on every change so it can double
|
||||||
|
* as a deck.gl `updateTrigger` (identity-compared).
|
||||||
|
*
|
||||||
|
* Writes are optimistic: `markClicked` updates local state synchronously so the pin recolours
|
||||||
|
* the instant it is clicked, independent of the PocketBase round-trip or the next listings fetch.
|
||||||
|
*/
|
||||||
|
export function useClickedListings() {
|
||||||
|
const [clickedUrls, setClickedUrls] = useState<Set<string>>(() => new Set());
|
||||||
|
const [userId, setUserId] = useState<string | null>(currentUserId);
|
||||||
|
|
||||||
|
const userIdRef = useRef(userId);
|
||||||
|
userIdRef.current = userId;
|
||||||
|
// Mirrors the latest committed set so markClicked can decide (synchronously) whether a url is
|
||||||
|
// new — without depending on the async state updater having run yet.
|
||||||
|
const clickedUrlsRef = useRef(clickedUrls);
|
||||||
|
clickedUrlsRef.current = clickedUrls;
|
||||||
|
|
||||||
|
// Follow login/logout via the shared authStore (also covers cross-tab auth changes).
|
||||||
|
useEffect(() => {
|
||||||
|
const unsubscribe = pb.authStore.onChange(() => setUserId(currentUserId()));
|
||||||
|
return unsubscribe;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Load the user's previously opened listings whenever the signed-in user changes. Signed-out
|
||||||
|
// users keep an in-memory set for the current session only (nothing to persist to).
|
||||||
|
useEffect(() => {
|
||||||
|
if (!userId) {
|
||||||
|
setClickedUrls(new Set());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
pb.collection(CLICKED_LISTINGS_COLLECTION)
|
||||||
|
.getFullList({ filter: `user = "${userId}"`, fields: 'url' })
|
||||||
|
.then((records) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
const urls = records
|
||||||
|
.map((record) => (record as { url?: unknown }).url)
|
||||||
|
.filter((url): url is string => typeof url === 'string');
|
||||||
|
setClickedUrls(new Set(urls));
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
// Visited history is a convenience; a failed load just starts the session empty.
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [userId]);
|
||||||
|
|
||||||
|
const markClicked = useCallback((url: string | undefined | null) => {
|
||||||
|
if (!url || clickedUrlsRef.current.has(url)) return; // unknown or already visited
|
||||||
|
|
||||||
|
// Optimistic, synchronous local update → the map pin recolours on click.
|
||||||
|
setClickedUrls((prev) => (prev.has(url) ? prev : new Set(prev).add(url)));
|
||||||
|
|
||||||
|
const uid = userIdRef.current;
|
||||||
|
if (!uid) return; // anonymous: recolour for the session, nothing to persist
|
||||||
|
pb.collection(CLICKED_LISTINGS_COLLECTION)
|
||||||
|
.create({ user: uid, url })
|
||||||
|
.catch(() => {
|
||||||
|
// Ignore duplicate-index conflicts and transient failures — local state already
|
||||||
|
// reflects the click, and a missed write only means it won't persist to the next visit.
|
||||||
|
});
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return { clickedUrls, markClicked };
|
||||||
|
}
|
||||||
|
|
@ -17,6 +17,8 @@ import type {
|
||||||
import {
|
import {
|
||||||
DENSITY_GRADIENT,
|
DENSITY_GRADIENT,
|
||||||
DENSITY_GRADIENT_DARK,
|
DENSITY_GRADIENT_DARK,
|
||||||
|
FILTERED_OUT_FILL,
|
||||||
|
FILTERED_OUT_LINE,
|
||||||
getEnumPaletteForFeature,
|
getEnumPaletteForFeature,
|
||||||
getFeatureGradient,
|
getFeatureGradient,
|
||||||
} from '../lib/consts';
|
} from '../lib/consts';
|
||||||
|
|
@ -125,11 +127,12 @@ export function useDeckLayers({
|
||||||
zoom,
|
zoom,
|
||||||
isDark,
|
isDark,
|
||||||
});
|
});
|
||||||
const { listingLayers, listingPopup, clearListingPopup } = useListingLayers({
|
const { listingLayers, listingPopup, clearListingPopup, clickedListingUrls, markListingClicked } =
|
||||||
listings: actualListings,
|
useListingLayers({
|
||||||
zoom,
|
listings: actualListings,
|
||||||
isDark,
|
zoom,
|
||||||
});
|
isDark,
|
||||||
|
});
|
||||||
const { developmentLayers, developmentPopup, clearDevelopmentPopup } = useDevelopmentLayers({
|
const { developmentLayers, developmentPopup, clearDevelopmentPopup } = useDevelopmentLayers({
|
||||||
developments,
|
developments,
|
||||||
zoom,
|
zoom,
|
||||||
|
|
@ -350,7 +353,12 @@ export function useDeckLayers({
|
||||||
getHexagon: (d) => d.h3,
|
getHexagon: (d) => d.h3,
|
||||||
getFillColor: (d) => {
|
getFillColor: (d) => {
|
||||||
if ((d.count as number) <= 0) {
|
if ((d.count as number) <= 0) {
|
||||||
return [0, 0, 0, 0] as [number, number, number, number];
|
// Filtered out: keep it on the map as a faint grey ghost (still
|
||||||
|
// clickable) rather than hiding it entirely.
|
||||||
|
return withColorOpacity(
|
||||||
|
FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
|
||||||
|
colorOpacityRef.current
|
||||||
|
);
|
||||||
}
|
}
|
||||||
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
|
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
|
||||||
const dark = isDarkRef.current;
|
const dark = isDarkRef.current;
|
||||||
|
|
@ -504,7 +512,12 @@ export function useDeckLayers({
|
||||||
getFillColor: (f) => {
|
getFillColor: (f) => {
|
||||||
const d = f.properties;
|
const d = f.properties;
|
||||||
if (d.count <= 0) {
|
if (d.count <= 0) {
|
||||||
return [0, 0, 0, 0] as [number, number, number, number];
|
// Filtered out: keep it on the map as a faint grey ghost (still
|
||||||
|
// clickable) rather than hiding it entirely.
|
||||||
|
return withColorOpacity(
|
||||||
|
FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
|
||||||
|
colorOpacityRef.current
|
||||||
|
);
|
||||||
}
|
}
|
||||||
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
|
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
|
||||||
const dark = isDarkRef.current;
|
const dark = isDarkRef.current;
|
||||||
|
|
@ -587,7 +600,7 @@ export function useDeckLayers({
|
||||||
return [29, 228, 195, 255] as [number, number, number, number];
|
return [29, 228, 195, 255] as [number, number, number, number];
|
||||||
if (pc === hoveredPostcodeRef.current)
|
if (pc === hoveredPostcodeRef.current)
|
||||||
return [29, 228, 195, 200] as [number, number, number, number];
|
return [29, 228, 195, 200] as [number, number, number, number];
|
||||||
if (f.properties.count <= 0) return [0, 0, 0, 0] as [number, number, number, number];
|
if (f.properties.count <= 0) return FILTERED_OUT_LINE[dark ? 'dark' : 'light'];
|
||||||
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
|
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
|
||||||
number,
|
number,
|
||||||
number,
|
number,
|
||||||
|
|
@ -599,7 +612,7 @@ export function useDeckLayers({
|
||||||
const pc = f.properties.postcode;
|
const pc = f.properties.postcode;
|
||||||
if (pc === selectedPostcodeIdRef.current) return 4;
|
if (pc === selectedPostcodeIdRef.current) return 4;
|
||||||
if (pc === hoveredPostcodeRef.current) return 2;
|
if (pc === hoveredPostcodeRef.current) return 2;
|
||||||
if (f.properties.count <= 0) return 0;
|
// Filtered-out polygons (count <= 0) keep the same 1px ghost border.
|
||||||
return 1;
|
return 1;
|
||||||
},
|
},
|
||||||
lineWidthUnits: 'pixels',
|
lineWidthUnits: 'pixels',
|
||||||
|
|
@ -749,6 +762,8 @@ export function useDeckLayers({
|
||||||
clearPopupInfo,
|
clearPopupInfo,
|
||||||
listingPopup,
|
listingPopup,
|
||||||
clearListingPopup,
|
clearListingPopup,
|
||||||
|
clickedListingUrls,
|
||||||
|
markListingClicked,
|
||||||
developmentPopup,
|
developmentPopup,
|
||||||
clearDevelopmentPopup,
|
clearDevelopmentPopup,
|
||||||
hoverPosition,
|
hoverPosition,
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,15 @@ import {
|
||||||
getSpecificCrimeFilterKeyId,
|
getSpecificCrimeFilterKeyId,
|
||||||
normalizeSpecificCrimeFilters,
|
normalizeSpecificCrimeFilters,
|
||||||
} from '../lib/crime-filter';
|
} from '../lib/crime-filter';
|
||||||
|
import {
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES,
|
||||||
|
createCrimeSeverityFilterKey,
|
||||||
|
getCrimeSeverityFeatureName,
|
||||||
|
getCrimeSeverityFilterKeyId,
|
||||||
|
getDefaultCrimeSeverityFeatureName,
|
||||||
|
normalizeCrimeSeverityFilters,
|
||||||
|
type CrimeSeverityFilterName,
|
||||||
|
} from '../lib/crime-severity-filter';
|
||||||
import {
|
import {
|
||||||
ELECTION_VOTE_SHARE_FILTER_NAME,
|
ELECTION_VOTE_SHARE_FILTER_NAME,
|
||||||
createElectionVoteShareFilterKey,
|
createElectionVoteShareFilterKey,
|
||||||
|
|
@ -33,6 +42,22 @@ import {
|
||||||
getEthnicityFilterKeyId,
|
getEthnicityFilterKeyId,
|
||||||
normalizeEthnicityFilters,
|
normalizeEthnicityFilters,
|
||||||
} from '../lib/ethnicity-filter';
|
} from '../lib/ethnicity-filter';
|
||||||
|
import {
|
||||||
|
QUALIFICATIONS_FILTER_NAME,
|
||||||
|
createQualificationFilterKey,
|
||||||
|
getDefaultQualificationFeatureName,
|
||||||
|
getQualificationFeatureName,
|
||||||
|
getQualificationFilterKeyId,
|
||||||
|
normalizeQualificationFilters,
|
||||||
|
} from '../lib/qualification-filter';
|
||||||
|
import {
|
||||||
|
TENURE_FILTER_NAME,
|
||||||
|
createTenureFilterKey,
|
||||||
|
getDefaultTenureFeatureName,
|
||||||
|
getTenureFeatureName,
|
||||||
|
getTenureFilterKeyId,
|
||||||
|
normalizeTenureFilters,
|
||||||
|
} from '../lib/tenure-filter';
|
||||||
import {
|
import {
|
||||||
POI_FILTER_NAMES,
|
POI_FILTER_NAMES,
|
||||||
createPoiFilterKey,
|
createPoiFilterKey,
|
||||||
|
|
@ -55,9 +80,15 @@ interface UseFiltersOptions {
|
||||||
|
|
||||||
function normalizeFilters(filters: FeatureFilters): FeatureFilters {
|
function normalizeFilters(filters: FeatureFilters): FeatureFilters {
|
||||||
return normalizePoiDistanceFilters(
|
return normalizePoiDistanceFilters(
|
||||||
normalizeEthnicityFilters(
|
normalizeTenureFilters(
|
||||||
normalizeElectionVoteShareFilters(
|
normalizeQualificationFilters(
|
||||||
normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters))
|
normalizeEthnicityFilters(
|
||||||
|
normalizeElectionVoteShareFilters(
|
||||||
|
normalizeCrimeSeverityFilters(
|
||||||
|
normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
@ -67,8 +98,11 @@ function getBackendFeatureName(name: string): string {
|
||||||
return (
|
return (
|
||||||
getSchoolBackendFeatureName(name) ??
|
getSchoolBackendFeatureName(name) ??
|
||||||
getSpecificCrimeFeatureName(name) ??
|
getSpecificCrimeFeatureName(name) ??
|
||||||
|
getCrimeSeverityFeatureName(name) ??
|
||||||
getElectionVoteShareFeatureName(name) ??
|
getElectionVoteShareFeatureName(name) ??
|
||||||
getEthnicityFeatureName(name) ??
|
getEthnicityFeatureName(name) ??
|
||||||
|
getQualificationFeatureName(name) ??
|
||||||
|
getTenureFeatureName(name) ??
|
||||||
getPoiDistanceFeatureName(name) ??
|
getPoiDistanceFeatureName(name) ??
|
||||||
name
|
name
|
||||||
);
|
);
|
||||||
|
|
@ -140,12 +174,21 @@ export function useFilters({
|
||||||
const specificCrimeFilterIdRef = useRef(
|
const specificCrimeFilterIdRef = useRef(
|
||||||
getNextNumericKeyId(initialFiltersRef.current!, getSpecificCrimeFilterKeyId)
|
getNextNumericKeyId(initialFiltersRef.current!, getSpecificCrimeFilterKeyId)
|
||||||
);
|
);
|
||||||
|
const crimeSeverityFilterIdRef = useRef(
|
||||||
|
getNextNumericKeyId(initialFiltersRef.current!, getCrimeSeverityFilterKeyId)
|
||||||
|
);
|
||||||
const electionVoteShareFilterIdRef = useRef(
|
const electionVoteShareFilterIdRef = useRef(
|
||||||
getNextNumericKeyId(initialFiltersRef.current!, getElectionVoteShareFilterKeyId)
|
getNextNumericKeyId(initialFiltersRef.current!, getElectionVoteShareFilterKeyId)
|
||||||
);
|
);
|
||||||
const ethnicityFilterIdRef = useRef(
|
const ethnicityFilterIdRef = useRef(
|
||||||
getNextNumericKeyId(initialFiltersRef.current!, getEthnicityFilterKeyId)
|
getNextNumericKeyId(initialFiltersRef.current!, getEthnicityFilterKeyId)
|
||||||
);
|
);
|
||||||
|
const qualificationFilterIdRef = useRef(
|
||||||
|
getNextNumericKeyId(initialFiltersRef.current!, getQualificationFilterKeyId)
|
||||||
|
);
|
||||||
|
const tenureFilterIdRef = useRef(
|
||||||
|
getNextNumericKeyId(initialFiltersRef.current!, getTenureFilterKeyId)
|
||||||
|
);
|
||||||
const poiDistanceFilterIdRef = useRef(
|
const poiDistanceFilterIdRef = useRef(
|
||||||
getNextNumericKeyId(initialFiltersRef.current!, getPoiDistanceFilterKeyId)
|
getNextNumericKeyId(initialFiltersRef.current!, getPoiDistanceFilterKeyId)
|
||||||
);
|
);
|
||||||
|
|
@ -184,8 +227,11 @@ export function useFilters({
|
||||||
if (
|
if (
|
||||||
name !== SCHOOL_FILTER_NAME &&
|
name !== SCHOOL_FILTER_NAME &&
|
||||||
name !== SPECIFIC_CRIMES_FILTER_NAME &&
|
name !== SPECIFIC_CRIMES_FILTER_NAME &&
|
||||||
|
!CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) &&
|
||||||
name !== ELECTION_VOTE_SHARE_FILTER_NAME &&
|
name !== ELECTION_VOTE_SHARE_FILTER_NAME &&
|
||||||
name !== ETHNICITIES_FILTER_NAME &&
|
name !== ETHNICITIES_FILTER_NAME &&
|
||||||
|
name !== QUALIFICATIONS_FILTER_NAME &&
|
||||||
|
name !== TENURE_FILTER_NAME &&
|
||||||
!POI_FILTER_NAMES.includes(name as PoiFilterName) &&
|
!POI_FILTER_NAMES.includes(name as PoiFilterName) &&
|
||||||
!meta
|
!meta
|
||||||
) {
|
) {
|
||||||
|
|
@ -201,8 +247,11 @@ export function useFilters({
|
||||||
const addsNewKey =
|
const addsNewKey =
|
||||||
name === SCHOOL_FILTER_NAME ||
|
name === SCHOOL_FILTER_NAME ||
|
||||||
name === SPECIFIC_CRIMES_FILTER_NAME ||
|
name === SPECIFIC_CRIMES_FILTER_NAME ||
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) ||
|
||||||
name === ELECTION_VOTE_SHARE_FILTER_NAME ||
|
name === ELECTION_VOTE_SHARE_FILTER_NAME ||
|
||||||
name === ETHNICITIES_FILTER_NAME ||
|
name === ETHNICITIES_FILTER_NAME ||
|
||||||
|
name === QUALIFICATIONS_FILTER_NAME ||
|
||||||
|
name === TENURE_FILTER_NAME ||
|
||||||
POI_FILTER_NAMES.includes(name as PoiFilterName) ||
|
POI_FILTER_NAMES.includes(name as PoiFilterName) ||
|
||||||
!(name in current);
|
!(name in current);
|
||||||
if (addsNewKey && Object.keys(current).length >= limit) {
|
if (addsNewKey && Object.keys(current).length >= limit) {
|
||||||
|
|
@ -247,6 +296,29 @@ export function useFilters({
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
|
||||||
|
const severityFilterName = name as CrimeSeverityFilterName;
|
||||||
|
const defaultSeverityFeatureName = getDefaultCrimeSeverityFeatureName(
|
||||||
|
features,
|
||||||
|
severityFilterName
|
||||||
|
);
|
||||||
|
const defaultSeverityFeature = defaultSeverityFeatureName
|
||||||
|
? features.find((feature) => feature.name === defaultSeverityFeatureName)
|
||||||
|
: undefined;
|
||||||
|
if (!defaultSeverityFeatureName) return prev;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...prev,
|
||||||
|
[createCrimeSeverityFilterKey(
|
||||||
|
severityFilterName,
|
||||||
|
defaultSeverityFeatureName,
|
||||||
|
crimeSeverityFilterIdRef.current++
|
||||||
|
)]: [
|
||||||
|
defaultSeverityFeature?.histogram?.min ?? defaultSeverityFeature?.min ?? 0,
|
||||||
|
defaultSeverityFeature?.histogram?.max ?? defaultSeverityFeature?.max ?? 100,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
|
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
|
||||||
const defaultVoteShareFeatureName = getDefaultElectionVoteShareFeatureName(features);
|
const defaultVoteShareFeatureName = getDefaultElectionVoteShareFeatureName(features);
|
||||||
const defaultVoteShareFeature = defaultVoteShareFeatureName
|
const defaultVoteShareFeature = defaultVoteShareFeatureName
|
||||||
|
|
@ -281,6 +353,41 @@ export function useFilters({
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (name === QUALIFICATIONS_FILTER_NAME) {
|
||||||
|
const defaultQualificationFeatureName = getDefaultQualificationFeatureName(features);
|
||||||
|
const defaultQualificationFeature = defaultQualificationFeatureName
|
||||||
|
? features.find((feature) => feature.name === defaultQualificationFeatureName)
|
||||||
|
: undefined;
|
||||||
|
if (!defaultQualificationFeatureName) return prev;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...prev,
|
||||||
|
[createQualificationFilterKey(
|
||||||
|
defaultQualificationFeatureName,
|
||||||
|
qualificationFilterIdRef.current++
|
||||||
|
)]: [
|
||||||
|
defaultQualificationFeature?.histogram?.min ?? defaultQualificationFeature?.min ?? 0,
|
||||||
|
defaultQualificationFeature?.histogram?.max ??
|
||||||
|
defaultQualificationFeature?.max ??
|
||||||
|
100,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (name === TENURE_FILTER_NAME) {
|
||||||
|
const defaultTenureFeatureName = getDefaultTenureFeatureName(features);
|
||||||
|
const defaultTenureFeature = defaultTenureFeatureName
|
||||||
|
? features.find((feature) => feature.name === defaultTenureFeatureName)
|
||||||
|
: undefined;
|
||||||
|
if (!defaultTenureFeatureName) return prev;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...prev,
|
||||||
|
[createTenureFilterKey(defaultTenureFeatureName, tenureFilterIdRef.current++)]: [
|
||||||
|
defaultTenureFeature?.histogram?.min ?? defaultTenureFeature?.min ?? 0,
|
||||||
|
defaultTenureFeature?.histogram?.max ?? defaultTenureFeature?.max ?? 100,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
|
||||||
const poiFilterName = name as PoiFilterName;
|
const poiFilterName = name as PoiFilterName;
|
||||||
const defaultPoiFeatureName = getDefaultPoiFilterFeatureName(features, poiFilterName);
|
const defaultPoiFeatureName = getDefaultPoiFilterFeatureName(features, poiFilterName);
|
||||||
|
|
@ -381,6 +488,23 @@ export function useFilters({
|
||||||
if (replaced) return normalizeFilters(next);
|
if (replaced) return normalizeFilters(next);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const crimeSeverityKeyId = getCrimeSeverityFilterKeyId(name);
|
||||||
|
if (crimeSeverityKeyId != null) {
|
||||||
|
let replaced = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
for (const [existingName, existingValue] of Object.entries(prev)) {
|
||||||
|
if (getCrimeSeverityFilterKeyId(existingName) === crimeSeverityKeyId) {
|
||||||
|
if (!replaced) {
|
||||||
|
next[name] = value;
|
||||||
|
replaced = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[existingName] = existingValue;
|
||||||
|
}
|
||||||
|
if (replaced) return normalizeFilters(next);
|
||||||
|
}
|
||||||
|
|
||||||
const ethnicityKeyId = getEthnicityFilterKeyId(name);
|
const ethnicityKeyId = getEthnicityFilterKeyId(name);
|
||||||
if (ethnicityKeyId != null) {
|
if (ethnicityKeyId != null) {
|
||||||
let replaced = false;
|
let replaced = false;
|
||||||
|
|
@ -398,6 +522,40 @@ export function useFilters({
|
||||||
if (replaced) return normalizeFilters(next);
|
if (replaced) return normalizeFilters(next);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const qualificationKeyId = getQualificationFilterKeyId(name);
|
||||||
|
if (qualificationKeyId != null) {
|
||||||
|
let replaced = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
for (const [existingName, existingValue] of Object.entries(prev)) {
|
||||||
|
if (getQualificationFilterKeyId(existingName) === qualificationKeyId) {
|
||||||
|
if (!replaced) {
|
||||||
|
next[name] = value;
|
||||||
|
replaced = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[existingName] = existingValue;
|
||||||
|
}
|
||||||
|
if (replaced) return normalizeFilters(next);
|
||||||
|
}
|
||||||
|
|
||||||
|
const tenureKeyId = getTenureFilterKeyId(name);
|
||||||
|
if (tenureKeyId != null) {
|
||||||
|
let replaced = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
for (const [existingName, existingValue] of Object.entries(prev)) {
|
||||||
|
if (getTenureFilterKeyId(existingName) === tenureKeyId) {
|
||||||
|
if (!replaced) {
|
||||||
|
next[name] = value;
|
||||||
|
replaced = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[existingName] = existingValue;
|
||||||
|
}
|
||||||
|
if (replaced) return normalizeFilters(next);
|
||||||
|
}
|
||||||
|
|
||||||
const electionVoteShareKeyId = getElectionVoteShareFilterKeyId(name);
|
const electionVoteShareKeyId = getElectionVoteShareFilterKeyId(name);
|
||||||
if (electionVoteShareKeyId != null) {
|
if (electionVoteShareKeyId != null) {
|
||||||
let replaced = false;
|
let replaced = false;
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,14 @@ import type {
|
||||||
HexagonPropertiesResponse,
|
HexagonPropertiesResponse,
|
||||||
HexagonStatsResponse,
|
HexagonStatsResponse,
|
||||||
} from '../types';
|
} from '../types';
|
||||||
import { buildFilterString, apiUrl, assertOk, logNonAbortError, authHeaders } from '../lib/api';
|
import {
|
||||||
|
buildFilterString,
|
||||||
|
apiUrl,
|
||||||
|
assertOk,
|
||||||
|
logNonAbortError,
|
||||||
|
authHeaders,
|
||||||
|
fetchCrimeRecords,
|
||||||
|
} from '../lib/api';
|
||||||
import { findOverlappingSelectableHexagon } from '../lib/h3-selection';
|
import { findOverlappingSelectableHexagon } from '../lib/h3-selection';
|
||||||
import { SMALLEST_VISIBLE_HEXAGON_RESOLUTION } from '../lib/consts';
|
import { SMALLEST_VISIBLE_HEXAGON_RESOLUTION } from '../lib/consts';
|
||||||
import type { TravelTimeEntry } from './useTravelTime';
|
import type { TravelTimeEntry } from './useTravelTime';
|
||||||
|
|
@ -327,6 +334,23 @@ export function useHexagonSelection({
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Lazily fetch a page of individual crimes for the current selection. The
|
||||||
|
// records are an attribute of the area (filter-independent), so this carries
|
||||||
|
// no filter string — only the selection identity and share code.
|
||||||
|
const loadCrimeRecords = useCallback(
|
||||||
|
(offset: number) => {
|
||||||
|
if (!selectedHexagon) {
|
||||||
|
return Promise.resolve({ records: [], total: 0, offset: 0, truncated: false });
|
||||||
|
}
|
||||||
|
const selection =
|
||||||
|
selectedHexagon.type === 'postcode'
|
||||||
|
? { postcode: selectedHexagon.id }
|
||||||
|
: { h3: selectedHexagon.id, resolution: selectedHexagon.resolution };
|
||||||
|
return fetchCrimeRecords(selection, offset, shareCode);
|
||||||
|
},
|
||||||
|
[selectedHexagon, shareCode]
|
||||||
|
);
|
||||||
|
|
||||||
const handleHexagonClick = useCallback(
|
const handleHexagonClick = useCallback(
|
||||||
(id: string, isPostcode = false, geometry?: PostcodeGeometry) => {
|
(id: string, isPostcode = false, geometry?: PostcodeGeometry) => {
|
||||||
if (selectedHexagon?.id === id) {
|
if (selectedHexagon?.id === id) {
|
||||||
|
|
@ -806,6 +830,7 @@ export function useHexagonSelection({
|
||||||
handlePropertiesTabClick,
|
handlePropertiesTabClick,
|
||||||
handleLoadMoreProperties,
|
handleLoadMoreProperties,
|
||||||
handleCloseSelection,
|
handleCloseSelection,
|
||||||
|
loadCrimeRecords,
|
||||||
selectedPostcodeGeometry,
|
selectedPostcodeGeometry,
|
||||||
handleLocationSearch,
|
handleLocationSearch,
|
||||||
handleCurrentLocationSearch,
|
handleCurrentLocationSearch,
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
import { useState, useCallback } from 'react';
|
import { useState, useCallback } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import { apiUrl, authHeaders, assertOk } from '../lib/api';
|
import { apiUrl, authHeaders, assertOk } from '../lib/api';
|
||||||
import { trackEvent } from '../lib/analytics';
|
import { trackEvent } from '../lib/analytics';
|
||||||
|
|
||||||
export function useLicense() {
|
export function useLicense() {
|
||||||
|
const { t } = useTranslation();
|
||||||
const [checkingOut, setCheckingOut] = useState(false);
|
const [checkingOut, setCheckingOut] = useState(false);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
|
@ -25,13 +27,13 @@ export function useLicense() {
|
||||||
window.location.href = data.url;
|
window.location.href = data.url;
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Checkout failed';
|
const msg = err instanceof Error ? err.message : t('upgrade.checkoutFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setCheckingOut(false);
|
setCheckingOut(false);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
return { startCheckout, checkingOut, error };
|
return { startCheckout, checkingOut, error };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,13 @@ import Supercluster from 'supercluster';
|
||||||
|
|
||||||
import type { ActualListing } from '../types';
|
import type { ActualListing } from '../types';
|
||||||
import { trackEvent } from '../lib/analytics';
|
import { trackEvent } from '../lib/analytics';
|
||||||
|
import { useClickedListings } from './useClickedListings';
|
||||||
|
|
||||||
|
/** Default red pin vs. the violet used once a user has opened that listing. */
|
||||||
|
const LISTING_PIN_FILL: [number, number, number, number] = [231, 76, 60, 240];
|
||||||
|
const LISTING_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 240];
|
||||||
|
const EXPANDED_PIN_FILL: [number, number, number, number] = [231, 76, 60, 245];
|
||||||
|
const EXPANDED_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 245];
|
||||||
|
|
||||||
const PRICE_LABEL_MIN_ZOOM = 14;
|
const PRICE_LABEL_MIN_ZOOM = 14;
|
||||||
const ADDRESS_LABEL_MIN_ZOOM = 16;
|
const ADDRESS_LABEL_MIN_ZOOM = 16;
|
||||||
|
|
@ -113,6 +120,7 @@ function spiderfyPosition(
|
||||||
export function useListingLayers({ listings, zoom, isDark }: UseListingLayersProps) {
|
export function useListingLayers({ listings, zoom, isDark }: UseListingLayersProps) {
|
||||||
const [popupInfo, setPopupInfo] = useState<ListingPopupInfo | null>(null);
|
const [popupInfo, setPopupInfo] = useState<ListingPopupInfo | null>(null);
|
||||||
const [selectedCluster, setSelectedCluster] = useState<ListingClusterPoint | null>(null);
|
const [selectedCluster, setSelectedCluster] = useState<ListingClusterPoint | null>(null);
|
||||||
|
const { clickedUrls, markClicked } = useClickedListings();
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
// Each refetch returns a fresh listings array and rebuilds the cluster index, so a
|
// Each refetch returns a fresh listings array and rebuilds the cluster index, so a
|
||||||
|
|
@ -212,12 +220,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
[clearUnlockedPopup]
|
[clearUnlockedPopup]
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleClick = useCallback((info: PickingInfo<ActualListing>) => {
|
const handleClick = useCallback(
|
||||||
const url = info.object?.listing_url;
|
(info: PickingInfo<ActualListing>) => {
|
||||||
if (!url) return;
|
const url = info.object?.listing_url;
|
||||||
trackEvent('Actual Listing Click', { url });
|
if (!url) return;
|
||||||
window.open(url, '_blank', 'noopener,noreferrer');
|
markClicked(url);
|
||||||
}, []);
|
trackEvent('Actual Listing Click', { url });
|
||||||
|
window.open(url, '_blank', 'noopener,noreferrer');
|
||||||
|
},
|
||||||
|
[markClicked]
|
||||||
|
);
|
||||||
|
|
||||||
const handleHoverRef = useRef(handleHover);
|
const handleHoverRef = useRef(handleHover);
|
||||||
handleHoverRef.current = handleHover;
|
handleHoverRef.current = handleHover;
|
||||||
|
|
@ -244,12 +256,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
[clearUnlockedPopup]
|
[clearUnlockedPopup]
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleExpandedClick = useCallback((info: PickingInfo<ExpandedListingMarker>) => {
|
const handleExpandedClick = useCallback(
|
||||||
const url = info.object?.listing.listing_url;
|
(info: PickingInfo<ExpandedListingMarker>) => {
|
||||||
if (!url) return;
|
const url = info.object?.listing.listing_url;
|
||||||
trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' });
|
if (!url) return;
|
||||||
window.open(url, '_blank', 'noopener,noreferrer');
|
markClicked(url);
|
||||||
}, []);
|
trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' });
|
||||||
|
window.open(url, '_blank', 'noopener,noreferrer');
|
||||||
|
},
|
||||||
|
[markClicked]
|
||||||
|
);
|
||||||
|
|
||||||
const handleExpandedHoverRef = useRef(handleExpandedHover);
|
const handleExpandedHoverRef = useRef(handleExpandedHover);
|
||||||
handleExpandedHoverRef.current = handleExpandedHover;
|
handleExpandedHoverRef.current = handleExpandedHover;
|
||||||
|
|
@ -345,7 +361,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
getPosition: (d) => [d.lon, d.lat],
|
getPosition: (d) => [d.lon, d.lat],
|
||||||
getRadius: 7,
|
getRadius: 7,
|
||||||
radiusUnits: 'pixels',
|
radiusUnits: 'pixels',
|
||||||
getFillColor: [231, 76, 60, 240],
|
getFillColor: (d) =>
|
||||||
|
clickedUrls.has(d.listing_url) ? LISTING_PIN_VISITED_FILL : LISTING_PIN_FILL,
|
||||||
getLineColor: [255, 255, 255, 255],
|
getLineColor: [255, 255, 255, 255],
|
||||||
getLineWidth: 1.5,
|
getLineWidth: 1.5,
|
||||||
lineWidthUnits: 'pixels',
|
lineWidthUnits: 'pixels',
|
||||||
|
|
@ -355,8 +372,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
highlightColor: [29, 228, 195, 220],
|
highlightColor: [29, 228, 195, 220],
|
||||||
onHover: stableHover,
|
onHover: stableHover,
|
||||||
onClick: stableClick,
|
onClick: stableClick,
|
||||||
|
updateTriggers: { getFillColor: clickedUrls },
|
||||||
}),
|
}),
|
||||||
[visibleListings, stableHover, stableClick]
|
[visibleListings, clickedUrls, stableHover, stableClick]
|
||||||
);
|
);
|
||||||
|
|
||||||
const clusterShadowLayer = useMemo(
|
const clusterShadowLayer = useMemo(
|
||||||
|
|
@ -441,7 +459,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
getPosition: (d) => [d.lng, d.lat],
|
getPosition: (d) => [d.lng, d.lat],
|
||||||
getRadius: 6,
|
getRadius: 6,
|
||||||
radiusUnits: 'pixels',
|
radiusUnits: 'pixels',
|
||||||
getFillColor: [231, 76, 60, 245],
|
getFillColor: (d) =>
|
||||||
|
clickedUrls.has(d.listing.listing_url) ? EXPANDED_PIN_VISITED_FILL : EXPANDED_PIN_FILL,
|
||||||
getLineColor: [255, 255, 255, 255],
|
getLineColor: [255, 255, 255, 255],
|
||||||
getLineWidth: 1.5,
|
getLineWidth: 1.5,
|
||||||
lineWidthUnits: 'pixels',
|
lineWidthUnits: 'pixels',
|
||||||
|
|
@ -451,8 +470,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
highlightColor: [29, 228, 195, 220],
|
highlightColor: [29, 228, 195, 220],
|
||||||
onHover: stableExpandedHover,
|
onHover: stableExpandedHover,
|
||||||
onClick: stableExpandedClick,
|
onClick: stableExpandedClick,
|
||||||
|
updateTriggers: { getFillColor: clickedUrls },
|
||||||
}),
|
}),
|
||||||
[expandedListings, stableExpandedHover, stableExpandedClick]
|
[expandedListings, clickedUrls, stableExpandedHover, stableExpandedClick]
|
||||||
);
|
);
|
||||||
|
|
||||||
const priceLabelLayer = useMemo(() => {
|
const priceLabelLayer = useMemo(() => {
|
||||||
|
|
@ -544,5 +564,11 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
|
||||||
setSelectedCluster(null);
|
setSelectedCluster(null);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return { listingLayers, listingPopup: popupInfo, clearListingPopup };
|
return {
|
||||||
|
listingLayers,
|
||||||
|
listingPopup: popupInfo,
|
||||||
|
clearListingPopup,
|
||||||
|
clickedListingUrls: clickedUrls,
|
||||||
|
markListingClicked: markClicked,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,11 @@ import {
|
||||||
} from '../lib/api';
|
} from '../lib/api';
|
||||||
import { getSchoolBackendFeatureName } from '../lib/school-filter';
|
import { getSchoolBackendFeatureName } from '../lib/school-filter';
|
||||||
import { getSpecificCrimeFeatureName } from '../lib/crime-filter';
|
import { getSpecificCrimeFeatureName } from '../lib/crime-filter';
|
||||||
|
import { getCrimeSeverityFeatureName } from '../lib/crime-severity-filter';
|
||||||
import { getElectionVoteShareFeatureName } from '../lib/election-filter';
|
import { getElectionVoteShareFeatureName } from '../lib/election-filter';
|
||||||
import { getEthnicityFeatureName } from '../lib/ethnicity-filter';
|
import { getEthnicityFeatureName } from '../lib/ethnicity-filter';
|
||||||
|
import { getQualificationFeatureName } from '../lib/qualification-filter';
|
||||||
|
import { getTenureFeatureName } from '../lib/tenure-filter';
|
||||||
import { getPoiDistanceFeatureName } from '../lib/poi-distance-filter';
|
import { getPoiDistanceFeatureName } from '../lib/poi-distance-filter';
|
||||||
import { POSTCODE_ZOOM_THRESHOLD } from '../lib/consts';
|
import { POSTCODE_ZOOM_THRESHOLD } from '../lib/consts';
|
||||||
import { COLOR_RANGE_LOW_PERCENTILE, COLOR_RANGE_HIGH_PERCENTILE } from '../lib/consts';
|
import { COLOR_RANGE_LOW_PERCENTILE, COLOR_RANGE_HIGH_PERCENTILE } from '../lib/consts';
|
||||||
|
|
@ -119,8 +122,11 @@ export function useMapData({
|
||||||
(name: string) =>
|
(name: string) =>
|
||||||
getSchoolBackendFeatureName(name) ??
|
getSchoolBackendFeatureName(name) ??
|
||||||
getSpecificCrimeFeatureName(name) ??
|
getSpecificCrimeFeatureName(name) ??
|
||||||
|
getCrimeSeverityFeatureName(name) ??
|
||||||
getElectionVoteShareFeatureName(name) ??
|
getElectionVoteShareFeatureName(name) ??
|
||||||
getEthnicityFeatureName(name) ??
|
getEthnicityFeatureName(name) ??
|
||||||
|
getQualificationFeatureName(name) ??
|
||||||
|
getTenureFeatureName(name) ??
|
||||||
getPoiDistanceFeatureName(name) ??
|
getPoiDistanceFeatureName(name) ??
|
||||||
name,
|
name,
|
||||||
[]
|
[]
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,19 @@
|
||||||
import { act, renderHook } from '@testing-library/react';
|
import { act, renderHook } from '@testing-library/react';
|
||||||
import { describe, expect, it } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import type { POI } from '../types';
|
import type { POI } from '../types';
|
||||||
import { usePoiLayers } from './usePoiLayers';
|
import { usePoiLayers } from './usePoiLayers';
|
||||||
|
|
||||||
|
vi.mock('react-i18next', () => ({
|
||||||
|
useTranslation: () => ({
|
||||||
|
t: (key: string) => {
|
||||||
|
if (key === 'common.places') return 'places';
|
||||||
|
if (key === 'map.poi.zoomInToSeeDetails') return 'Zoom in to see details';
|
||||||
|
return key;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
const supermarket: POI = {
|
const supermarket: POI = {
|
||||||
id: 'poi-1',
|
id: 'poi-1',
|
||||||
name: 'Market Hall',
|
name: 'Market Hall',
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import type { PickingInfo } from '@deck.gl/core';
|
import type { PickingInfo } from '@deck.gl/core';
|
||||||
import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers';
|
import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers';
|
||||||
import Supercluster from 'supercluster';
|
import Supercluster from 'supercluster';
|
||||||
|
|
@ -65,6 +66,7 @@ function getPoiIconSize(poi: POI): number {
|
||||||
}
|
}
|
||||||
|
|
||||||
export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
|
export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
|
||||||
|
const { t } = useTranslation();
|
||||||
const [popupInfo, setPopupInfo] = useState<PopupInfo | null>(null);
|
const [popupInfo, setPopupInfo] = useState<PopupInfo | null>(null);
|
||||||
|
|
||||||
// Dismiss a lingering hover/cluster popup once the POIs behind it are gone — e.g.
|
// Dismiss a lingering hover/cluster popup once the POIs behind it are gone — e.g.
|
||||||
|
|
@ -108,8 +110,8 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
|
||||||
setPopupInfo({
|
setPopupInfo({
|
||||||
x: info.x,
|
x: info.x,
|
||||||
y: info.y,
|
y: info.y,
|
||||||
name: `${info.object.count} places`,
|
name: `${info.object.count} ${t('common.places')}`,
|
||||||
category: 'Zoom in to see details',
|
category: t('map.poi.zoomInToSeeDetails'),
|
||||||
group: '',
|
group: '',
|
||||||
emoji: '',
|
emoji: '',
|
||||||
id: '',
|
id: '',
|
||||||
|
|
@ -119,7 +121,7 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
|
||||||
} else {
|
} else {
|
||||||
setPopupInfo(null);
|
setPopupInfo(null);
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const handleClusterHoverRef = useRef(handleClusterHover);
|
const handleClusterHoverRef = useRef(handleClusterHover);
|
||||||
handleClusterHoverRef.current = handleClusterHover;
|
handleClusterHoverRef.current = handleClusterHover;
|
||||||
|
|
|
||||||
80
frontend/src/hooks/useRevealOnExpand.test.tsx
Normal file
80
frontend/src/hooks/useRevealOnExpand.test.tsx
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
import { cleanup, fireEvent, render } from '@testing-library/react';
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { useState } from 'react';
|
||||||
|
|
||||||
|
import { useRevealOnExpand } from './useRevealOnExpand';
|
||||||
|
|
||||||
|
function GroupList() {
|
||||||
|
const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
|
||||||
|
const [expanded, setExpanded] = useState(false);
|
||||||
|
return (
|
||||||
|
<div ref={setContainer} data-testid="container">
|
||||||
|
<div ref={registerGroup('g1')} data-testid="group">
|
||||||
|
<button
|
||||||
|
data-testid="toggle"
|
||||||
|
onClick={() => {
|
||||||
|
const willExpand = !expanded;
|
||||||
|
setExpanded(willExpand);
|
||||||
|
onToggle('g1', willExpand);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
toggle
|
||||||
|
</button>
|
||||||
|
{expanded && <div style={{ height: 1000 }} />}
|
||||||
|
</div>
|
||||||
|
<div style={{ height: 2000 }} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('useRevealOnExpand', () => {
|
||||||
|
const original = {
|
||||||
|
resizeObserver: globalThis.ResizeObserver,
|
||||||
|
scrollTo: HTMLElement.prototype.scrollTo,
|
||||||
|
getBoundingClientRect: HTMLElement.prototype.getBoundingClientRect,
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
globalThis.ResizeObserver = class {
|
||||||
|
observe() {}
|
||||||
|
unobserve() {}
|
||||||
|
disconnect() {}
|
||||||
|
} as unknown as typeof ResizeObserver;
|
||||||
|
HTMLElement.prototype.scrollTo = vi.fn();
|
||||||
|
// The opened group's bottom (800) sits below the container's bottom (500),
|
||||||
|
// so it needs a 300px downward scroll to come fully into view.
|
||||||
|
HTMLElement.prototype.getBoundingClientRect = function (this: HTMLElement) {
|
||||||
|
const id = this.dataset?.testid;
|
||||||
|
const bottom = id === 'group' ? 800 : id === 'container' ? 500 : 0;
|
||||||
|
return { bottom } as DOMRect;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
globalThis.ResizeObserver = original.resizeObserver;
|
||||||
|
HTMLElement.prototype.scrollTo = original.scrollTo;
|
||||||
|
HTMLElement.prototype.getBoundingClientRect = original.getBoundingClientRect;
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('scrolls a freshly expanded group into view', () => {
|
||||||
|
const view = render(<GroupList />);
|
||||||
|
const scrollTo = view.getByTestId('container').scrollTo as ReturnType<typeof vi.fn>;
|
||||||
|
|
||||||
|
fireEvent.click(view.getByTestId('toggle'));
|
||||||
|
|
||||||
|
expect(scrollTo).toHaveBeenCalledWith({ top: 300 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not scroll when a group is collapsed', () => {
|
||||||
|
const view = render(<GroupList />);
|
||||||
|
const scrollTo = view.getByTestId('container').scrollTo as ReturnType<typeof vi.fn>;
|
||||||
|
|
||||||
|
fireEvent.click(view.getByTestId('toggle')); // expand
|
||||||
|
scrollTo.mockClear();
|
||||||
|
fireEvent.click(view.getByTestId('toggle')); // collapse
|
||||||
|
|
||||||
|
expect(scrollTo).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
97
frontend/src/hooks/useRevealOnExpand.ts
Normal file
97
frontend/src/hooks/useRevealOnExpand.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
||||||
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-scrolls a freshly expanded collapsible group into view inside a scroll
|
||||||
|
* container, so opening a group near the bottom reveals its content without
|
||||||
|
* manual scrolling. The (sticky) group header stays pinned at the top.
|
||||||
|
*
|
||||||
|
* Wiring:
|
||||||
|
* const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
|
||||||
|
* <div ref={setContainer} className="overflow-y-auto">
|
||||||
|
* {groups.map((g) => (
|
||||||
|
* <div key={g.name} ref={registerGroup(g.name)}>
|
||||||
|
* <Header
|
||||||
|
* onToggle={() => {
|
||||||
|
* const willExpand = !isExpanded(g.name);
|
||||||
|
* toggle(g.name);
|
||||||
|
* onToggle(g.name, willExpand);
|
||||||
|
* }}
|
||||||
|
* />
|
||||||
|
* {isExpanded(g.name) && <Body />}
|
||||||
|
* </div>
|
||||||
|
* ))}
|
||||||
|
* </div>
|
||||||
|
*
|
||||||
|
* When the container element is also driven by another ref (e.g. the callback
|
||||||
|
* ref from useRetainedScrollTop), compose them by calling both inside a single
|
||||||
|
* callback ref.
|
||||||
|
*/
|
||||||
|
export function useRevealOnExpand<T extends HTMLElement = HTMLDivElement>() {
|
||||||
|
const containerRef = useRef<T | null>(null);
|
||||||
|
const groupRefs = useRef<Map<string, T>>(new Map());
|
||||||
|
const groupRefCallbacks = useRef<Map<string, (node: T | null) => void>>(new Map());
|
||||||
|
// A fresh object each toggle so re-expanding the same group still re-triggers.
|
||||||
|
const [groupToReveal, setGroupToReveal] = useState<{ name: string } | null>(null);
|
||||||
|
|
||||||
|
const setContainer = useCallback((node: T | null) => {
|
||||||
|
containerRef.current = node;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Returns a stable ref callback per group name so React doesn't churn the map
|
||||||
|
// (cleanup + re-register) on every render.
|
||||||
|
const registerGroup = useCallback((name: string) => {
|
||||||
|
const existing = groupRefCallbacks.current.get(name);
|
||||||
|
if (existing) return existing;
|
||||||
|
const cb = (node: T | null) => {
|
||||||
|
if (node) groupRefs.current.set(name, node);
|
||||||
|
else groupRefs.current.delete(name);
|
||||||
|
};
|
||||||
|
groupRefCallbacks.current.set(name, cb);
|
||||||
|
return cb;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Call from a group's toggle handler: reveals the group when it is being
|
||||||
|
// expanded, and cancels any pending reveal when it is being collapsed.
|
||||||
|
const onToggle = useCallback((name: string, willExpand: boolean) => {
|
||||||
|
setGroupToReveal(willExpand ? { name } : null);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!groupToReveal) return;
|
||||||
|
const el = groupRefs.current.get(groupToReveal.name);
|
||||||
|
const container = containerRef.current;
|
||||||
|
if (!el || !container) return;
|
||||||
|
|
||||||
|
// Scroll down just enough to bring the bottom of the opened group into view.
|
||||||
|
// For groups taller than the pane this scrolls past the header, but it stays
|
||||||
|
// visible via its sticky positioning. Never scroll up (group already in view).
|
||||||
|
const alignBottom = () => {
|
||||||
|
const delta = el.getBoundingClientRect().bottom - container.getBoundingClientRect().bottom;
|
||||||
|
if (delta <= 1) return;
|
||||||
|
container.scrollTo({ top: container.scrollTop + delta });
|
||||||
|
};
|
||||||
|
|
||||||
|
alignBottom();
|
||||||
|
|
||||||
|
// The group's charts and async data (e.g. nearby stations) can grow its
|
||||||
|
// height after this first pass, so keep the bottom pinned as it settles.
|
||||||
|
// Stop once the user scrolls or after a short grace period so we never fight
|
||||||
|
// a deliberate scroll.
|
||||||
|
let timer = 0;
|
||||||
|
const observer = new ResizeObserver(alignBottom);
|
||||||
|
const stop = () => {
|
||||||
|
observer.disconnect();
|
||||||
|
container.removeEventListener('wheel', stop);
|
||||||
|
container.removeEventListener('touchmove', stop);
|
||||||
|
window.clearTimeout(timer);
|
||||||
|
};
|
||||||
|
observer.observe(el);
|
||||||
|
container.addEventListener('wheel', stop, { passive: true });
|
||||||
|
container.addEventListener('touchmove', stop, { passive: true });
|
||||||
|
timer = window.setTimeout(stop, 1500);
|
||||||
|
|
||||||
|
return stop;
|
||||||
|
}, [groupToReveal]);
|
||||||
|
|
||||||
|
return { setContainer, registerGroup, onToggle };
|
||||||
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { useState, useCallback, useRef, useEffect } from 'react';
|
import { useState, useCallback, useRef, useEffect } from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import pb from '../lib/pocketbase';
|
import pb from '../lib/pocketbase';
|
||||||
import { apiUrl, authHeaders } from '../lib/api';
|
import { apiUrl, authHeaders } from '../lib/api';
|
||||||
import { trackEvent } from '../lib/analytics';
|
import { trackEvent } from '../lib/analytics';
|
||||||
|
|
@ -24,6 +25,7 @@ function nextPollDelay(attempt: number): number {
|
||||||
}
|
}
|
||||||
|
|
||||||
export function useSavedSearches(userId: string | null) {
|
export function useSavedSearches(userId: string | null) {
|
||||||
|
const { t } = useTranslation();
|
||||||
const [searches, setSearches] = useState<SavedSearch[]>([]);
|
const [searches, setSearches] = useState<SavedSearch[]>([]);
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
const [saving, setSaving] = useState(false);
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
@ -127,11 +129,11 @@ export function useSavedSearches(userId: string | null) {
|
||||||
stopPolling();
|
stopPolling();
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : 'Failed to load searches');
|
setError(err instanceof Error ? err.message : t('savedPage.loadFailed'));
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}, [userId, fetchRecords, startPolling, stopPolling]);
|
}, [userId, fetchRecords, startPolling, stopPolling, t]);
|
||||||
|
|
||||||
const saveSearch = useCallback(
|
const saveSearch = useCallback(
|
||||||
async (name: string, paramsOverride?: string) => {
|
async (name: string, paramsOverride?: string) => {
|
||||||
|
|
@ -169,14 +171,14 @@ export function useSavedSearches(userId: string | null) {
|
||||||
console.warn('Background screenshot failed:', err);
|
console.warn('Background screenshot failed:', err);
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Failed to save search';
|
const msg = err instanceof Error ? err.message : t('savedPage.saveFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setSaving(false);
|
setSaving(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[userId, fetchSearches]
|
[userId, fetchSearches, t]
|
||||||
);
|
);
|
||||||
|
|
||||||
const deleteSearch = useCallback(async (id: string) => {
|
const deleteSearch = useCallback(async (id: string) => {
|
||||||
|
|
@ -186,27 +188,27 @@ export function useSavedSearches(userId: string | null) {
|
||||||
trackEvent('Search Delete');
|
trackEvent('Search Delete');
|
||||||
setSearches((prev) => prev.filter((s) => s.id !== id));
|
setSearches((prev) => prev.filter((s) => s.id !== id));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : 'Failed to delete search');
|
setError(err instanceof Error ? err.message : t('savedPage.deleteFailed'));
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const updateSearchNotes = useCallback(async (id: string, notes: string) => {
|
const updateSearchNotes = useCallback(async (id: string, notes: string) => {
|
||||||
try {
|
try {
|
||||||
await pb.collection('saved_searches').update(id, { notes });
|
await pb.collection('saved_searches').update(id, { notes });
|
||||||
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, notes } : s)));
|
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, notes } : s)));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : 'Failed to update notes');
|
setError(err instanceof Error ? err.message : t('savedPage.updateNotesFailed'));
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const updateSearchName = useCallback(async (id: string, name: string) => {
|
const updateSearchName = useCallback(async (id: string, name: string) => {
|
||||||
try {
|
try {
|
||||||
await pb.collection('saved_searches').update(id, { name });
|
await pb.collection('saved_searches').update(id, { name });
|
||||||
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, name } : s)));
|
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, name } : s)));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : 'Failed to update name');
|
setError(err instanceof Error ? err.message : t('savedPage.updateNameFailed'));
|
||||||
}
|
}
|
||||||
}, []);
|
}, [t]);
|
||||||
|
|
||||||
const updateSearchParams = useCallback(
|
const updateSearchParams = useCallback(
|
||||||
async (id: string, params: string) => {
|
async (id: string, params: string) => {
|
||||||
|
|
@ -238,14 +240,14 @@ export function useSavedSearches(userId: string | null) {
|
||||||
console.warn('Background screenshot failed:', err);
|
console.warn('Background screenshot failed:', err);
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const msg = err instanceof Error ? err.message : 'Failed to update search';
|
const msg = err instanceof Error ? err.message : t('savedPage.updateFailed');
|
||||||
setError(msg);
|
setError(msg);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
setSaving(false);
|
setSaving(false);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[userId, fetchSearches]
|
[userId, fetchSearches, t]
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -50,6 +50,13 @@ export function useTutorial(initialLoading: boolean, isMobile: boolean, blocked
|
||||||
placement: 'left' as const,
|
placement: 'left' as const,
|
||||||
skipBeacon: true,
|
skipBeacon: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
target: '[data-tutorial="overlays-button"]',
|
||||||
|
title: t('tutorial.step7Title'),
|
||||||
|
content: t('tutorial.step7Content'),
|
||||||
|
placement: 'left' as const,
|
||||||
|
skipBeacon: true,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
[t]
|
[t]
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { createRoot } from 'react-dom/client';
|
import { createRoot } from 'react-dom/client';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
import App from './App';
|
import App from './App';
|
||||||
import { i18nReady } from './i18n';
|
import { i18nReady } from './i18n';
|
||||||
import { BugsinkErrorBoundary, initBugsink } from './lib/bugsink';
|
import { BugsinkErrorBoundary, initBugsink } from './lib/bugsink';
|
||||||
|
|
@ -14,12 +15,17 @@ if (!container) {
|
||||||
const root = container;
|
const root = container;
|
||||||
|
|
||||||
function AppErrorFallback() {
|
function AppErrorFallback() {
|
||||||
|
const { t } = useTranslation();
|
||||||
return (
|
return (
|
||||||
<div className="flex min-h-screen items-center justify-center bg-warm-50 px-6 text-center text-warm-900 dark:bg-navy-950 dark:text-warm-100">
|
<div className="flex min-h-screen items-center justify-center bg-warm-50 px-6 text-center text-warm-900 dark:bg-navy-950 dark:text-warm-100">
|
||||||
<div>
|
<div>
|
||||||
<h1 className="text-xl font-semibold">Something went wrong</h1>
|
<h1 className="text-xl font-semibold">
|
||||||
|
{t('errors.appCrashTitle', { defaultValue: 'Something went wrong' })}
|
||||||
|
</h1>
|
||||||
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
|
<p className="mt-2 text-sm text-warm-600 dark:text-warm-300">
|
||||||
Refresh the page to try again.
|
{t('errors.appCrashBody', {
|
||||||
|
defaultValue: 'Refresh the page to try again.',
|
||||||
|
})}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ import { createSchoolFilterKey } from './school-filter';
|
||||||
import { createSpecificCrimeFilterKey } from './crime-filter';
|
import { createSpecificCrimeFilterKey } from './crime-filter';
|
||||||
import { createElectionVoteShareFilterKey } from './election-filter';
|
import { createElectionVoteShareFilterKey } from './election-filter';
|
||||||
import { createEthnicityFilterKey } from './ethnicity-filter';
|
import { createEthnicityFilterKey } from './ethnicity-filter';
|
||||||
|
import { createQualificationFilterKey } from './qualification-filter';
|
||||||
|
import { createTenureFilterKey } from './tenure-filter';
|
||||||
import {
|
import {
|
||||||
POI_COUNT_2KM_FILTER_NAME,
|
POI_COUNT_2KM_FILTER_NAME,
|
||||||
TRANSPORT_DISTANCE_FILTER_NAME,
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
|
@ -98,19 +100,19 @@ describe('api utilities', () => {
|
||||||
|
|
||||||
it('serializes specific crime filters using their selected backend crime feature', () => {
|
it('serializes specific crime filters using their selected backend crime feature', () => {
|
||||||
const features: FeatureMeta[] = [
|
const features: FeatureMeta[] = [
|
||||||
{ name: 'Burglary (avg/yr)', type: 'numeric', min: 0, max: 20 },
|
{ name: 'Burglary (/yr, 7y)', type: 'numeric', min: 0, max: 20 },
|
||||||
{ name: 'Vehicle crime (avg/yr)', type: 'numeric', min: 0, max: 30 },
|
{ name: 'Vehicle crime (/yr, 7y)', type: 'numeric', min: 0, max: 30 },
|
||||||
];
|
];
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
buildFilterString(
|
buildFilterString(
|
||||||
{
|
{
|
||||||
[createSpecificCrimeFilterKey('Burglary (avg/yr)', 1)]: [0, 5],
|
[createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1)]: [0, 5],
|
||||||
[createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2)]: [1, 10],
|
[createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2)]: [1, 10],
|
||||||
},
|
},
|
||||||
features
|
features
|
||||||
)
|
)
|
||||||
).toBe('Burglary (avg/yr):0:5;;Vehicle crime (avg/yr):1:10');
|
).toBe('Burglary (/yr, 7y):0:5;;Vehicle crime (/yr, 7y):1:10');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('serializes election vote-share filters using their selected backend party feature', () => {
|
it('serializes election vote-share filters using their selected backend party feature', () => {
|
||||||
|
|
@ -144,6 +146,40 @@ describe('api utilities', () => {
|
||||||
).toBe('% White:20:80');
|
).toBe('% White:20:80');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('serializes qualification filters using their selected backend band feature', () => {
|
||||||
|
const features: FeatureMeta[] = [
|
||||||
|
{ name: '% Degree or higher', type: 'numeric', min: 0, max: 100 },
|
||||||
|
{ name: '% No qualifications', type: 'numeric', min: 0, max: 100 },
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(
|
||||||
|
buildFilterString(
|
||||||
|
{
|
||||||
|
[createQualificationFilterKey('% Degree or higher', 1)]: [20, 60],
|
||||||
|
[createQualificationFilterKey('% No qualifications', 2)]: [0, 25],
|
||||||
|
},
|
||||||
|
features
|
||||||
|
)
|
||||||
|
).toBe('% Degree or higher:20:60;;% No qualifications:0:25');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('serializes tenure filters using their selected backend band feature', () => {
|
||||||
|
const features: FeatureMeta[] = [
|
||||||
|
{ name: '% Owner occupied', type: 'numeric', min: 0, max: 100 },
|
||||||
|
{ name: '% Private rent', type: 'numeric', min: 0, max: 100 },
|
||||||
|
];
|
||||||
|
|
||||||
|
expect(
|
||||||
|
buildFilterString(
|
||||||
|
{
|
||||||
|
[createTenureFilterKey('% Owner occupied', 1)]: [20, 60],
|
||||||
|
[createTenureFilterKey('% Private rent', 2)]: [0, 25],
|
||||||
|
},
|
||||||
|
features
|
||||||
|
)
|
||||||
|
).toBe('% Owner occupied:20:60;;% Private rent:0:25');
|
||||||
|
});
|
||||||
|
|
||||||
it('serializes amenity distance filters using their selected backend feature', () => {
|
it('serializes amenity distance filters using their selected backend feature', () => {
|
||||||
const features: FeatureMeta[] = [
|
const features: FeatureMeta[] = [
|
||||||
{ name: 'Distance to nearest amenity (Park) (km)', type: 'numeric', min: 0, max: 2 },
|
{ name: 'Distance to nearest amenity (Park) (km)', type: 'numeric', min: 0, max: 2 },
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
import type { FeatureMeta, FeatureFilters } from '../types';
|
import type { FeatureMeta, FeatureFilters, CrimeRecordsResponse } from '../types';
|
||||||
import { INITIAL_RETRY_MS, MAX_RETRY_MS } from './consts';
|
import { INITIAL_RETRY_MS, MAX_RETRY_MS } from './consts';
|
||||||
import pb from './pocketbase';
|
import pb from './pocketbase';
|
||||||
import { getSchoolBackendFeatureName } from './school-filter';
|
import { getSchoolBackendFeatureName } from './school-filter';
|
||||||
import { getSpecificCrimeFeatureName } from './crime-filter';
|
import { getSpecificCrimeFeatureName } from './crime-filter';
|
||||||
|
import { getCrimeSeverityFeatureName } from './crime-severity-filter';
|
||||||
import { getElectionVoteShareFeatureName } from './election-filter';
|
import { getElectionVoteShareFeatureName } from './election-filter';
|
||||||
import { getEthnicityFeatureName } from './ethnicity-filter';
|
import { getEthnicityFeatureName } from './ethnicity-filter';
|
||||||
|
import { getQualificationFeatureName } from './qualification-filter';
|
||||||
|
import { getTenureFeatureName } from './tenure-filter';
|
||||||
import { getPoiDistanceFeatureName } from './poi-distance-filter';
|
import { getPoiDistanceFeatureName } from './poi-distance-filter';
|
||||||
|
|
||||||
const SCREENSHOT_LANGUAGES = new Set(['en', 'fr', 'de', 'zh', 'hi', 'hu']);
|
const SCREENSHOT_LANGUAGES = new Set(['en', 'fr', 'de', 'zh', 'hi', 'hu']);
|
||||||
|
|
@ -61,6 +64,7 @@ const DEMO_GATED_ENDPOINTS = new Set([
|
||||||
'postcode-stats',
|
'postcode-stats',
|
||||||
'postcode-properties',
|
'postcode-properties',
|
||||||
'hexagon-properties',
|
'hexagon-properties',
|
||||||
|
'crime-records',
|
||||||
'journey',
|
'journey',
|
||||||
'actual-listings',
|
'actual-listings',
|
||||||
'developments',
|
'developments',
|
||||||
|
|
@ -186,6 +190,26 @@ export async function shortenUrl(params: string, language?: string): Promise<str
|
||||||
return `${window.location.origin}${data.url}`;
|
return `${window.location.origin}${data.url}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Fetch one page of individual crime records for a hexagon or a postcode. The
|
||||||
|
* list is independent of property filters, so no filter string is sent. */
|
||||||
|
export async function fetchCrimeRecords(
|
||||||
|
selection: { h3: string; resolution: number } | { postcode: string },
|
||||||
|
offset: number,
|
||||||
|
shareCode?: string
|
||||||
|
): Promise<CrimeRecordsResponse> {
|
||||||
|
const params = new URLSearchParams({ offset: offset.toString() });
|
||||||
|
if ('postcode' in selection) {
|
||||||
|
params.set('postcode', selection.postcode);
|
||||||
|
} else {
|
||||||
|
params.set('h3', selection.h3);
|
||||||
|
params.set('resolution', selection.resolution.toString());
|
||||||
|
}
|
||||||
|
if (shareCode) params.set('share', shareCode);
|
||||||
|
const res = await fetch(apiUrl('crime-records', params), authHeaders());
|
||||||
|
assertOk(res, 'crime-records');
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
export function buildFilterString(
|
export function buildFilterString(
|
||||||
filters: FeatureFilters,
|
filters: FeatureFilters,
|
||||||
features: FeatureMeta[],
|
features: FeatureMeta[],
|
||||||
|
|
@ -200,8 +224,11 @@ export function buildFilterString(
|
||||||
const backendName =
|
const backendName =
|
||||||
getSchoolBackendFeatureName(name) ??
|
getSchoolBackendFeatureName(name) ??
|
||||||
getSpecificCrimeFeatureName(name) ??
|
getSpecificCrimeFeatureName(name) ??
|
||||||
|
getCrimeSeverityFeatureName(name) ??
|
||||||
getElectionVoteShareFeatureName(name) ??
|
getElectionVoteShareFeatureName(name) ??
|
||||||
getEthnicityFeatureName(name) ??
|
getEthnicityFeatureName(name) ??
|
||||||
|
getQualificationFeatureName(name) ??
|
||||||
|
getTenureFeatureName(name) ??
|
||||||
getPoiDistanceFeatureName(name) ??
|
getPoiDistanceFeatureName(name) ??
|
||||||
name;
|
name;
|
||||||
const prev = merged.get(backendName);
|
const prev = merged.get(backendName);
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ export const SMALLEST_VISIBLE_HEXAGON_RESOLUTION = Math.max(
|
||||||
// past the finest hexagon level so detail appears while still relatively
|
// past the finest hexagon level so detail appears while still relatively
|
||||||
// zoomed out. (Each overlay additionally can't render below its own tile-data
|
// zoomed out. (Each overlay additionally can't render below its own tile-data
|
||||||
// floor, OVERLAY_MIN_ZOOM, regardless of this limit.)
|
// floor, OVERLAY_MIN_ZOOM, regardless of this limit.)
|
||||||
export const POSTCODE_ZOOM_THRESHOLD = 14;
|
export const POSTCODE_ZOOM_THRESHOLD = 13.5;
|
||||||
export const POSTCODE_SEARCH_ZOOM = 16;
|
export const POSTCODE_SEARCH_ZOOM = 16;
|
||||||
|
|
||||||
export const FEATURE_GRADIENT: { t: number; color: [number, number, number] }[] = [
|
export const FEATURE_GRADIENT: { t: number; color: [number, number, number] }[] = [
|
||||||
|
|
@ -264,7 +264,7 @@ export const STACKED_GROUPS: Record<
|
||||||
label: string;
|
label: string;
|
||||||
/** If set, use this feature's stats for the total and info popup. Otherwise sum components. */
|
/** If set, use this feature's stats for the total and info popup. Otherwise sum components. */
|
||||||
feature?: string;
|
feature?: string;
|
||||||
/** If set, display this feature's mean as the primary value (e.g. per-1k rate) instead of the absolute total. */
|
/** If set, display this feature's mean as the primary value (e.g. a percentage) instead of the absolute total. */
|
||||||
rateFeature?: string;
|
rateFeature?: string;
|
||||||
/** Suffix shown after the total value (e.g. "avg/yr") */
|
/** Suffix shown after the total value (e.g. "avg/yr") */
|
||||||
unit?: string;
|
unit?: string;
|
||||||
|
|
@ -275,30 +275,30 @@ export const STACKED_GROUPS: Record<
|
||||||
Crime: [
|
Crime: [
|
||||||
{
|
{
|
||||||
label: 'Serious crime',
|
label: 'Serious crime',
|
||||||
feature: 'Serious crime (avg/yr)',
|
feature: 'Serious crime (/yr, 7y)',
|
||||||
unit: '',
|
unit: '',
|
||||||
components: [
|
components: [
|
||||||
'Violence and sexual offences (avg/yr)',
|
'Violence and sexual offences (/yr, 7y)',
|
||||||
'Robbery (avg/yr)',
|
'Robbery (/yr, 7y)',
|
||||||
'Burglary (avg/yr)',
|
'Burglary (/yr, 7y)',
|
||||||
'Possession of weapons (avg/yr)',
|
'Possession of weapons (/yr, 7y)',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: 'Minor crime',
|
label: 'Minor crime',
|
||||||
feature: 'Minor crime (avg/yr)',
|
feature: 'Minor crime (/yr, 7y)',
|
||||||
unit: '',
|
unit: '',
|
||||||
components: [
|
components: [
|
||||||
'Anti-social behaviour (avg/yr)',
|
'Anti-social behaviour (/yr, 7y)',
|
||||||
'Criminal damage and arson (avg/yr)',
|
'Criminal damage and arson (/yr, 7y)',
|
||||||
'Shoplifting (avg/yr)',
|
'Shoplifting (/yr, 7y)',
|
||||||
'Bicycle theft (avg/yr)',
|
'Bicycle theft (/yr, 7y)',
|
||||||
'Theft from the person (avg/yr)',
|
'Theft from the person (/yr, 7y)',
|
||||||
'Other theft (avg/yr)',
|
'Other theft (/yr, 7y)',
|
||||||
'Vehicle crime (avg/yr)',
|
'Vehicle crime (/yr, 7y)',
|
||||||
'Public order (avg/yr)',
|
'Public order (/yr, 7y)',
|
||||||
'Drugs (avg/yr)',
|
'Drugs (/yr, 7y)',
|
||||||
'Other crime (avg/yr)',
|
'Other crime (/yr, 7y)',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
@ -493,20 +493,20 @@ export function getEnumValueColor(
|
||||||
|
|
||||||
/** Explicit colors for stacked bar segments. */
|
/** Explicit colors for stacked bar segments. */
|
||||||
export const STACKED_SEGMENT_COLORS: Record<string, string> = {
|
export const STACKED_SEGMENT_COLORS: Record<string, string> = {
|
||||||
'Violence and sexual offences (avg/yr)': '#ef4444',
|
'Violence and sexual offences (/yr, 7y)': '#ef4444',
|
||||||
'Robbery (avg/yr)': '#f97316',
|
'Robbery (/yr, 7y)': '#f97316',
|
||||||
'Burglary (avg/yr)': '#eab308',
|
'Burglary (/yr, 7y)': '#eab308',
|
||||||
'Possession of weapons (avg/yr)': '#8b5cf6',
|
'Possession of weapons (/yr, 7y)': '#8b5cf6',
|
||||||
'Anti-social behaviour (avg/yr)': '#14b8a6',
|
'Anti-social behaviour (/yr, 7y)': '#14b8a6',
|
||||||
'Criminal damage and arson (avg/yr)': '#f97316',
|
'Criminal damage and arson (/yr, 7y)': '#f97316',
|
||||||
'Shoplifting (avg/yr)': '#ec4899',
|
'Shoplifting (/yr, 7y)': '#ec4899',
|
||||||
'Bicycle theft (avg/yr)': '#22c55e',
|
'Bicycle theft (/yr, 7y)': '#22c55e',
|
||||||
'Theft from the person (avg/yr)': '#d946ef',
|
'Theft from the person (/yr, 7y)': '#d946ef',
|
||||||
'Other theft (avg/yr)': '#06b6d4',
|
'Other theft (/yr, 7y)': '#06b6d4',
|
||||||
'Vehicle crime (avg/yr)': '#3b82f6',
|
'Vehicle crime (/yr, 7y)': '#3b82f6',
|
||||||
'Public order (avg/yr)': '#8b5cf6',
|
'Public order (/yr, 7y)': '#8b5cf6',
|
||||||
'Drugs (avg/yr)': '#22c55e',
|
'Drugs (/yr, 7y)': '#22c55e',
|
||||||
'Other crime (avg/yr)': '#6b7280',
|
'Other crime (/yr, 7y)': '#6b7280',
|
||||||
'% White': '#3b82f6',
|
'% White': '#3b82f6',
|
||||||
'% South Asian': '#f97316',
|
'% South Asian': '#f97316',
|
||||||
'% East Asian': '#eab308',
|
'% East Asian': '#eab308',
|
||||||
|
|
|
||||||
79
frontend/src/lib/crime-filter.test.ts
Normal file
79
frontend/src/lib/crime-filter.test.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import type { FeatureMeta } from '../types';
|
||||||
|
import {
|
||||||
|
SPECIFIC_CRIME_FEATURE_NAMES,
|
||||||
|
createSpecificCrimeFilterKey,
|
||||||
|
getDefaultSpecificCrimeFeatureName,
|
||||||
|
getSpecificCrimeFeatureName,
|
||||||
|
getSpecificCrimeFilterKeyId,
|
||||||
|
getSpecificCrimeType,
|
||||||
|
getSpecificCrimeWindow,
|
||||||
|
isSpecificCrimeFeatureName,
|
||||||
|
isSpecificCrimeFilterName,
|
||||||
|
normalizeSpecificCrimeFilters,
|
||||||
|
specificCrimeFeatureName,
|
||||||
|
withSpecificCrimeWindow,
|
||||||
|
} from './crime-filter';
|
||||||
|
|
||||||
|
const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
|
||||||
|
|
||||||
|
describe('specific-crime window helpers', () => {
|
||||||
|
it('recognizes both the 7-year and 2-year rate features', () => {
|
||||||
|
expect(isSpecificCrimeFeatureName('Burglary (/yr, 7y)')).toBe(true);
|
||||||
|
expect(isSpecificCrimeFeatureName('Burglary (/yr, 2y)')).toBe(true);
|
||||||
|
expect(isSpecificCrimeFeatureName('Burglary (avg/yr)')).toBe(false);
|
||||||
|
// The "Serious crime" / "Minor crime" aggregates stay separate sliders.
|
||||||
|
expect(isSpecificCrimeFeatureName('Serious crime (/yr, 7y)')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extracts the window and bare type', () => {
|
||||||
|
expect(getSpecificCrimeWindow('Burglary (/yr, 2y)')).toBe('2y');
|
||||||
|
expect(getSpecificCrimeWindow('Burglary (/yr, 7y)')).toBe('7y');
|
||||||
|
expect(getSpecificCrimeWindow('Burglary')).toBeNull();
|
||||||
|
expect(getSpecificCrimeType('Violence and sexual offences (/yr, 2y)')).toBe(
|
||||||
|
'Violence and sexual offences'
|
||||||
|
);
|
||||||
|
expect(getSpecificCrimeType('not a crime')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('swaps a feature name between windows and round-trips', () => {
|
||||||
|
expect(withSpecificCrimeWindow('Burglary (/yr, 7y)', '2y')).toBe(
|
||||||
|
'Burglary (/yr, 2y)'
|
||||||
|
);
|
||||||
|
expect(withSpecificCrimeWindow('Burglary (/yr, 2y)', '7y')).toBe(
|
||||||
|
'Burglary (/yr, 7y)'
|
||||||
|
);
|
||||||
|
// Unrecognized names pass through untouched.
|
||||||
|
expect(withSpecificCrimeWindow('Burglary', '2y')).toBe('Burglary');
|
||||||
|
expect(specificCrimeFeatureName('Drugs', '2y')).toBe('Drugs (/yr, 2y)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('defaults to the 7-year window and enumerates 7-year dropdown names', () => {
|
||||||
|
const features = [numeric('Burglary (/yr, 7y)'), numeric('Burglary (/yr, 2y)')];
|
||||||
|
expect(getDefaultSpecificCrimeFeatureName(features)).toBe('Burglary (/yr, 7y)');
|
||||||
|
expect(SPECIFIC_CRIME_FEATURE_NAMES.every((n) => n.endsWith('(/yr, 7y)'))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('specific-crime filter keys', () => {
|
||||||
|
it('resolves a 2-year filter key back to its backend column', () => {
|
||||||
|
const key = createSpecificCrimeFilterKey('Burglary (/yr, 2y)', 3);
|
||||||
|
expect(isSpecificCrimeFilterName(key)).toBe(true);
|
||||||
|
expect(getSpecificCrimeFilterKeyId(key)).toBe('3');
|
||||||
|
expect(getSpecificCrimeFeatureName(key)).toBe('Burglary (/yr, 2y)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('folds a bare 2-year crime feature into a multi-instance key', () => {
|
||||||
|
const normalized = normalizeSpecificCrimeFilters({
|
||||||
|
'Burglary (/yr, 2y)': [1, 5],
|
||||||
|
'Median price (£)': [0, 100],
|
||||||
|
});
|
||||||
|
const keys = Object.keys(normalized);
|
||||||
|
expect(keys.some((k) => isSpecificCrimeFilterName(k))).toBe(true);
|
||||||
|
const crimeKey = keys.find((k) => isSpecificCrimeFilterName(k))!;
|
||||||
|
expect(getSpecificCrimeFeatureName(crimeKey)).toBe('Burglary (/yr, 2y)');
|
||||||
|
expect(normalized[crimeKey]).toEqual([1, 5]);
|
||||||
|
// Non-crime filters are left untouched.
|
||||||
|
expect(normalized['Median price (£)']).toEqual([0, 100]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -1,26 +1,74 @@
|
||||||
import type { FeatureFilters, FeatureMeta } from '../types';
|
import type { FeatureFilters, FeatureMeta } from '../types';
|
||||||
|
import type { VariantFilterConfig } from './variant-filter';
|
||||||
|
|
||||||
export const SPECIFIC_CRIMES_FILTER_NAME = 'Specific crimes';
|
export const SPECIFIC_CRIMES_FILTER_NAME = 'Specific crimes';
|
||||||
export const SPECIFIC_CRIMES_FILTER_KEY_PREFIX = `${SPECIFIC_CRIMES_FILTER_NAME}:`;
|
export const SPECIFIC_CRIMES_FILTER_KEY_PREFIX = `${SPECIFIC_CRIMES_FILTER_NAME}:`;
|
||||||
|
|
||||||
export const SPECIFIC_CRIME_FEATURE_NAMES = [
|
// The "pick one crime type" filter selects a single street-level category, with
|
||||||
'Violence and sexual offences (avg/yr)',
|
// a toggle to measure it over the 7-year or the recent 2-year window. The 7-year
|
||||||
'Burglary (avg/yr)',
|
// window is the default/primary one (also headlined in the area pane).
|
||||||
'Robbery (avg/yr)',
|
export const SPECIFIC_CRIME_TYPES = [
|
||||||
'Vehicle crime (avg/yr)',
|
'Violence and sexual offences',
|
||||||
'Anti-social behaviour (avg/yr)',
|
'Burglary',
|
||||||
'Criminal damage and arson (avg/yr)',
|
'Robbery',
|
||||||
'Other theft (avg/yr)',
|
'Vehicle crime',
|
||||||
'Theft from the person (avg/yr)',
|
'Anti-social behaviour',
|
||||||
'Shoplifting (avg/yr)',
|
'Criminal damage and arson',
|
||||||
'Bicycle theft (avg/yr)',
|
'Other theft',
|
||||||
'Drugs (avg/yr)',
|
'Theft from the person',
|
||||||
'Possession of weapons (avg/yr)',
|
'Shoplifting',
|
||||||
'Public order (avg/yr)',
|
'Bicycle theft',
|
||||||
'Other crime (avg/yr)',
|
'Drugs',
|
||||||
|
'Possession of weapons',
|
||||||
|
'Public order',
|
||||||
|
'Other crime',
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set<string>(SPECIFIC_CRIME_FEATURE_NAMES);
|
export const SPECIFIC_CRIME_WINDOW_DEFAULT = '7y';
|
||||||
|
export const SPECIFIC_CRIME_WINDOWS = ['7y', '2y'] as const;
|
||||||
|
export type SpecificCrimeWindow = (typeof SPECIFIC_CRIME_WINDOWS)[number];
|
||||||
|
|
||||||
|
/** The backend rate-feature column for a crime type in a given window. */
|
||||||
|
export function specificCrimeFeatureName(type: string, window: SpecificCrimeWindow): string {
|
||||||
|
return `${type} (/yr, ${window})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Canonical dropdown enumeration: one feature name per crime type, in the
|
||||||
|
// default (7-year) window. The window toggle re-points these to the chosen window.
|
||||||
|
export const SPECIFIC_CRIME_FEATURE_NAMES = SPECIFIC_CRIME_TYPES.map((type) =>
|
||||||
|
specificCrimeFeatureName(type, SPECIFIC_CRIME_WINDOW_DEFAULT)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Every recognized specific-crime feature, across all windows. Used to detect /
|
||||||
|
// resolve a filter key's backend column for either window.
|
||||||
|
const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set<string>(
|
||||||
|
SPECIFIC_CRIME_TYPES.flatMap((type) =>
|
||||||
|
SPECIFIC_CRIME_WINDOWS.map((window) => specificCrimeFeatureName(type, window))
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const SPECIFIC_CRIME_WINDOW_RE = / \(\/yr, (7y|2y)\)$/;
|
||||||
|
|
||||||
|
/** The window suffix of a specific-crime feature name (e.g. "2y"), or null. */
|
||||||
|
export function getSpecificCrimeWindow(featureName: string): SpecificCrimeWindow | null {
|
||||||
|
const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
|
||||||
|
return match ? (match[1] as SpecificCrimeWindow) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The bare crime type of a specific-crime feature name (e.g. "Burglary"), or null. */
|
||||||
|
export function getSpecificCrimeType(featureName: string): string | null {
|
||||||
|
const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
|
||||||
|
return match ? featureName.slice(0, match.index) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The same crime type's feature name in a different window (no-op if unrecognized). */
|
||||||
|
export function withSpecificCrimeWindow(
|
||||||
|
featureName: string,
|
||||||
|
window: SpecificCrimeWindow
|
||||||
|
): string {
|
||||||
|
const type = getSpecificCrimeType(featureName);
|
||||||
|
return type ? specificCrimeFeatureName(type, window) : featureName;
|
||||||
|
}
|
||||||
|
|
||||||
export function isSpecificCrimeFeatureName(name: string): boolean {
|
export function isSpecificCrimeFeatureName(name: string): boolean {
|
||||||
return SPECIFIC_CRIME_FEATURE_NAME_SET.has(name);
|
return SPECIFIC_CRIME_FEATURE_NAME_SET.has(name);
|
||||||
|
|
@ -101,7 +149,7 @@ export function getSpecificCrimeFilterMeta(features: FeatureMeta[]): FeatureMeta
|
||||||
description:
|
description:
|
||||||
'Violence, burglary, robbery, drugs, shoplifting, vehicle crime, anti-social behaviour, public order, theft, and other crime types',
|
'Violence, burglary, robbery, drugs, shoplifting, vehicle crime, anti-social behaviour, public order, theft, and other crime types',
|
||||||
detail:
|
detail:
|
||||||
'Filter by one street-level crime category at a time using an area-normalised crime density near each postcode (not a count of incidents per year).',
|
'Filter by one street-level crime category at a time, as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
|
||||||
source: 'crime',
|
source: 'crime',
|
||||||
suffix: '',
|
suffix: '',
|
||||||
};
|
};
|
||||||
|
|
@ -115,3 +163,26 @@ export function clampSpecificCrimeRange(
|
||||||
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
|
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
|
||||||
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
|
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const SPECIFIC_CRIME_VARIANT_CONFIG: VariantFilterConfig = {
|
||||||
|
filterName: SPECIFIC_CRIMES_FILTER_NAME,
|
||||||
|
featureNames: SPECIFIC_CRIME_FEATURE_NAMES,
|
||||||
|
dropdownLabelKey: 'filters.crimeType',
|
||||||
|
getFilterMeta: getSpecificCrimeFilterMeta,
|
||||||
|
getDefaultFeatureName: getDefaultSpecificCrimeFeatureName,
|
||||||
|
getFeatureName: getSpecificCrimeFeatureName,
|
||||||
|
replaceFilterKeySelection: replaceSpecificCrimeFilterKeySelection,
|
||||||
|
clampRange: clampSpecificCrimeRange,
|
||||||
|
// Dropdown shows the bare crime type; the window toggle carries the 7y/2y suffix.
|
||||||
|
getOptionLabelSource: (name) => getSpecificCrimeType(name) ?? name,
|
||||||
|
window: {
|
||||||
|
labelKey: 'filters.crimeWindow',
|
||||||
|
options: [
|
||||||
|
{ id: '7y', labelKey: 'filters.crimeWindow7y' },
|
||||||
|
{ id: '2y', labelKey: 'filters.crimeWindow2y' },
|
||||||
|
],
|
||||||
|
getWindow: getSpecificCrimeWindow,
|
||||||
|
withWindow: (name, windowId) =>
|
||||||
|
withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
|
||||||
85
frontend/src/lib/crime-severity-filter.test.ts
Normal file
85
frontend/src/lib/crime-severity-filter.test.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import type { FeatureMeta } from '../types';
|
||||||
|
import {
|
||||||
|
CRIME_SEVERITY_FILTER_NAMES,
|
||||||
|
createCrimeSeverityFilterKey,
|
||||||
|
getCrimeSeverityFeatureName,
|
||||||
|
getCrimeSeverityFilterKeyId,
|
||||||
|
getCrimeSeverityFilterName,
|
||||||
|
getCrimeSeverityVariantConfig,
|
||||||
|
getDefaultCrimeSeverityFeatureName,
|
||||||
|
isCrimeSeverityFeatureName,
|
||||||
|
isCrimeSeverityFilterName,
|
||||||
|
normalizeCrimeSeverityFilters,
|
||||||
|
} from './crime-severity-filter';
|
||||||
|
|
||||||
|
const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
|
||||||
|
|
||||||
|
describe('crime-severity feature recognition', () => {
|
||||||
|
it('recognizes Serious/Minor crime in both windows only', () => {
|
||||||
|
expect(isCrimeSeverityFeatureName('Serious crime (/yr, 7y)')).toBe(true);
|
||||||
|
expect(isCrimeSeverityFeatureName('Serious crime (/yr, 2y)')).toBe(true);
|
||||||
|
expect(isCrimeSeverityFeatureName('Minor crime (/yr, 2y)')).toBe(true);
|
||||||
|
// Detailed categories belong to "Specific crimes", not severity.
|
||||||
|
expect(isCrimeSeverityFeatureName('Burglary (/yr, 7y)')).toBe(false);
|
||||||
|
// Bare names without a window suffix are not feature columns.
|
||||||
|
expect(isCrimeSeverityFeatureName('Serious crime')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps a feature or key back to its severity filter name', () => {
|
||||||
|
expect(getCrimeSeverityFilterName('Serious crime (/yr, 2y)')).toBe('Serious crime');
|
||||||
|
expect(getCrimeSeverityFilterName('Minor crime (/yr, 7y)')).toBe('Minor crime');
|
||||||
|
const key = createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0);
|
||||||
|
expect(getCrimeSeverityFilterName(key)).toBe('Serious crime');
|
||||||
|
expect(getCrimeSeverityFilterName('Burglary (/yr, 7y)')).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('crime-severity filter keys', () => {
|
||||||
|
it('resolves a 2-year key back to its backend column and id', () => {
|
||||||
|
const key = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 4);
|
||||||
|
expect(isCrimeSeverityFilterName(key)).toBe(true);
|
||||||
|
expect(getCrimeSeverityFilterKeyId(key)).toBe('4');
|
||||||
|
expect(getCrimeSeverityFeatureName(key)).toBe('Minor crime (/yr, 2y)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('defaults to the 7-year window when present', () => {
|
||||||
|
const features = [
|
||||||
|
numeric('Serious crime (/yr, 7y)'),
|
||||||
|
numeric('Serious crime (/yr, 2y)'),
|
||||||
|
];
|
||||||
|
expect(getDefaultCrimeSeverityFeatureName(features, 'Serious crime')).toBe(
|
||||||
|
'Serious crime (/yr, 7y)'
|
||||||
|
);
|
||||||
|
expect(getDefaultCrimeSeverityFeatureName(features, 'Minor crime')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('folds bare severity features but leaves specific crimes and others alone', () => {
|
||||||
|
const normalized = normalizeCrimeSeverityFilters({
|
||||||
|
'Serious crime (/yr, 2y)': [1, 5],
|
||||||
|
'Burglary (/yr, 7y)': [0, 3],
|
||||||
|
'Median price (£)': [0, 100],
|
||||||
|
});
|
||||||
|
const keys = Object.keys(normalized);
|
||||||
|
const severityKey = keys.find((k) => isCrimeSeverityFilterName(k))!;
|
||||||
|
expect(getCrimeSeverityFeatureName(severityKey)).toBe('Serious crime (/yr, 2y)');
|
||||||
|
expect(normalized[severityKey]).toEqual([1, 5]);
|
||||||
|
// Specific-crime + plain features pass through untouched.
|
||||||
|
expect(normalized['Burglary (/yr, 7y)']).toEqual([0, 3]);
|
||||||
|
expect(normalized['Median price (£)']).toEqual([0, 100]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('crime-severity variant config', () => {
|
||||||
|
it('exposes a single variant per severity with a 7y/2y window toggle', () => {
|
||||||
|
for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
|
||||||
|
const config = getCrimeSeverityVariantConfig(filterName);
|
||||||
|
expect(config.filterName).toBe(filterName);
|
||||||
|
expect(config.featureNames).toEqual([`${filterName} (/yr, 7y)`]);
|
||||||
|
expect(config.window?.options.map((o) => o.id)).toEqual(['7y', '2y']);
|
||||||
|
// Switching window re-points to the same severity, other window.
|
||||||
|
const sevenYear = `${filterName} (/yr, 7y)`;
|
||||||
|
expect(config.window?.withWindow(sevenYear, '2y')).toBe(`${filterName} (/yr, 2y)`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
193
frontend/src/lib/crime-severity-filter.ts
Normal file
193
frontend/src/lib/crime-severity-filter.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
||||||
|
import type { FeatureFilters, FeatureMeta } from '../types';
|
||||||
|
import type { VariantFilterConfig } from './variant-filter';
|
||||||
|
import {
|
||||||
|
SPECIFIC_CRIME_WINDOW_DEFAULT,
|
||||||
|
clampSpecificCrimeRange,
|
||||||
|
getSpecificCrimeType,
|
||||||
|
getSpecificCrimeWindow,
|
||||||
|
specificCrimeFeatureName,
|
||||||
|
withSpecificCrimeWindow,
|
||||||
|
type SpecificCrimeWindow,
|
||||||
|
} from './crime-filter';
|
||||||
|
|
||||||
|
// "Serious crime" and "Minor crime" are aggregate severity rollups (they overlap
|
||||||
|
// the 14 detailed categories, so they must stay OUT of the "Specific crimes"
|
||||||
|
// dropdown). Each is a single feature with a 7-year/2-year window toggle — the
|
||||||
|
// same folding the specific-crimes card has, but with no variant dropdown (one
|
||||||
|
// variant). Modeled on the multi-filter-name POI pattern: one lib, two filter
|
||||||
|
// names, distinguished by their key prefix / bare crime type.
|
||||||
|
|
||||||
|
export const SERIOUS_CRIME_FILTER_NAME = 'Serious crime';
|
||||||
|
export const MINOR_CRIME_FILTER_NAME = 'Minor crime';
|
||||||
|
|
||||||
|
export const CRIME_SEVERITY_FILTER_NAMES = [
|
||||||
|
SERIOUS_CRIME_FILTER_NAME,
|
||||||
|
MINOR_CRIME_FILTER_NAME,
|
||||||
|
] as const;
|
||||||
|
export type CrimeSeverityFilterName = (typeof CRIME_SEVERITY_FILTER_NAMES)[number];
|
||||||
|
|
||||||
|
const CRIME_SEVERITY_TYPE_SET = new Set<string>(CRIME_SEVERITY_FILTER_NAMES);
|
||||||
|
|
||||||
|
const CRIME_SEVERITY_DETAILS: Record<CrimeSeverityFilterName, { description: string; detail: string }> = {
|
||||||
|
[SERIOUS_CRIME_FILTER_NAME]: {
|
||||||
|
description: 'Violence, robbery, burglary and weapons possession near the postcode',
|
||||||
|
detail:
|
||||||
|
'Combined count of the more serious street-level categories (violence and sexual offences, robbery, burglary, possession of weapons), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
|
||||||
|
},
|
||||||
|
[MINOR_CRIME_FILTER_NAME]: {
|
||||||
|
description: 'Anti-social behaviour, theft, criminal damage, drugs and public order near the postcode',
|
||||||
|
detail:
|
||||||
|
'Combined count of the lower-severity street-level categories (anti-social behaviour, theft, criminal damage and arson, drugs, public order), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function keyPrefix(filterName: CrimeSeverityFilterName): string {
|
||||||
|
return `${filterName}:`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The severity filter a name belongs to (from its key prefix or bare feature name), or null. */
|
||||||
|
export function getCrimeSeverityFilterName(name: string): CrimeSeverityFilterName | null {
|
||||||
|
for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
|
||||||
|
if (name.startsWith(keyPrefix(filterName))) return filterName;
|
||||||
|
}
|
||||||
|
const type = getSpecificCrimeType(name);
|
||||||
|
return type && CRIME_SEVERITY_TYPE_SET.has(type) ? (type as CrimeSeverityFilterName) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isCrimeSeverityFeatureName(name: string): boolean {
|
||||||
|
const type = getSpecificCrimeType(name);
|
||||||
|
return type != null && CRIME_SEVERITY_TYPE_SET.has(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isCrimeSeverityFilterName(name: string): boolean {
|
||||||
|
return getCrimeSeverityFilterName(name) != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createCrimeSeverityFilterKey(
|
||||||
|
filterName: CrimeSeverityFilterName,
|
||||||
|
featureName: string,
|
||||||
|
id: number | string
|
||||||
|
): string {
|
||||||
|
return `${keyPrefix(filterName)}${encodeURIComponent(featureName)}:${id}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCrimeSeverityFilterKeyId(name: string): string | null {
|
||||||
|
const filterName = getCrimeSeverityFilterName(name);
|
||||||
|
if (!filterName) return null;
|
||||||
|
const prefix = keyPrefix(filterName);
|
||||||
|
if (!name.startsWith(prefix)) return null; // a bare feature name has no id
|
||||||
|
const rest = name.substring(prefix.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
return lastColon === -1 ? null : rest.substring(lastColon + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseCrimeSeverityFilterKey(name: string): string | null {
|
||||||
|
const filterName = getCrimeSeverityFilterName(name);
|
||||||
|
if (!filterName) return null;
|
||||||
|
const prefix = keyPrefix(filterName);
|
||||||
|
if (!name.startsWith(prefix)) return null;
|
||||||
|
const rest = name.substring(prefix.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
if (lastColon === -1) return null;
|
||||||
|
|
||||||
|
const decoded = decodeURIComponent(rest.substring(0, lastColon));
|
||||||
|
return isCrimeSeverityFeatureName(decoded) ? decoded : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCrimeSeverityFeatureName(name: string): string | null {
|
||||||
|
if (isCrimeSeverityFeatureName(name)) return name;
|
||||||
|
return parseCrimeSeverityFilterKey(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function replaceCrimeSeverityFilterKeySelection(key: string, featureName: string): string {
|
||||||
|
const filterName = getCrimeSeverityFilterName(key) ?? getCrimeSeverityFilterName(featureName);
|
||||||
|
const id = getCrimeSeverityFilterKeyId(key) ?? '0';
|
||||||
|
// filterName is always resolvable here: the key carries a prefix and a window
|
||||||
|
// switch keeps the same severity type. Fall back defensively to the key as-is.
|
||||||
|
if (!filterName) return key;
|
||||||
|
return createCrimeSeverityFilterKey(filterName, featureName, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The default (7-year) backend feature for a severity, if present in `features`. */
|
||||||
|
export function getDefaultCrimeSeverityFeatureName(
|
||||||
|
features: FeatureMeta[],
|
||||||
|
filterName: CrimeSeverityFilterName
|
||||||
|
): string | null {
|
||||||
|
const name = specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT);
|
||||||
|
return features.some((feature) => feature.name === name) ? name : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeCrimeSeverityFilters(filters: FeatureFilters): FeatureFilters {
|
||||||
|
let changed = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
|
||||||
|
for (const [name, value] of Object.entries(filters)) {
|
||||||
|
if (isCrimeSeverityFeatureName(name)) {
|
||||||
|
const filterName = getCrimeSeverityFilterName(name)!;
|
||||||
|
next[createCrimeSeverityFilterKey(filterName, name, Object.keys(next).length)] = value;
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[name] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed ? next : filters;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getCrimeSeverityFilterMeta(
|
||||||
|
features: FeatureMeta[],
|
||||||
|
filterName: CrimeSeverityFilterName
|
||||||
|
): FeatureMeta {
|
||||||
|
const sourceFeatureName = getDefaultCrimeSeverityFeatureName(features, filterName);
|
||||||
|
const sourceFeature = sourceFeatureName
|
||||||
|
? features.find((feature) => feature.name === sourceFeatureName)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: filterName,
|
||||||
|
type: 'numeric',
|
||||||
|
group: 'Crime',
|
||||||
|
min: sourceFeature?.min ?? 0,
|
||||||
|
max: sourceFeature?.max ?? 100,
|
||||||
|
step: 1,
|
||||||
|
description: CRIME_SEVERITY_DETAILS[filterName].description,
|
||||||
|
detail: CRIME_SEVERITY_DETAILS[filterName].detail,
|
||||||
|
source: 'crime',
|
||||||
|
suffix: '',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clampCrimeSeverityRange(
|
||||||
|
value: [number, number],
|
||||||
|
feature?: FeatureMeta
|
||||||
|
): [number, number] {
|
||||||
|
return clampSpecificCrimeRange(value, feature);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The variant-filter config for one severity (single variant + 7y/2y window toggle). */
|
||||||
|
export function getCrimeSeverityVariantConfig(
|
||||||
|
filterName: CrimeSeverityFilterName
|
||||||
|
): VariantFilterConfig {
|
||||||
|
return {
|
||||||
|
filterName,
|
||||||
|
// One variant: the severity itself, in the default window. The card hides the
|
||||||
|
// (single-option) dropdown and exposes only the window toggle.
|
||||||
|
featureNames: [specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT)],
|
||||||
|
dropdownLabelKey: 'filters.crimeType',
|
||||||
|
getFilterMeta: (features) => getCrimeSeverityFilterMeta(features, filterName),
|
||||||
|
getDefaultFeatureName: (features) => getDefaultCrimeSeverityFeatureName(features, filterName),
|
||||||
|
getFeatureName: getCrimeSeverityFeatureName,
|
||||||
|
replaceFilterKeySelection: replaceCrimeSeverityFilterKeySelection,
|
||||||
|
clampRange: clampCrimeSeverityRange,
|
||||||
|
window: {
|
||||||
|
labelKey: 'filters.crimeWindow',
|
||||||
|
options: [
|
||||||
|
{ id: '7y', labelKey: 'filters.crimeWindow7y' },
|
||||||
|
{ id: '2y', labelKey: 'filters.crimeWindow2y' },
|
||||||
|
],
|
||||||
|
getWindow: getSpecificCrimeWindow,
|
||||||
|
withWindow: (name, windowId) =>
|
||||||
|
withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -6,24 +6,47 @@
|
||||||
|
|
||||||
export interface CrimeTypeDef {
|
export interface CrimeTypeDef {
|
||||||
value: string;
|
value: string;
|
||||||
|
/** English label (fallback / reference). */
|
||||||
label: string;
|
label: string;
|
||||||
|
/** i18n key under `crimeTypes.*` used to render the selector label. */
|
||||||
|
labelKey: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const CRIME_TYPES: readonly CrimeTypeDef[] = [
|
export const CRIME_TYPES: readonly CrimeTypeDef[] = [
|
||||||
{ value: 'Violence and sexual offences', label: 'Violence & sexual offences' },
|
{
|
||||||
{ value: 'Anti-social behaviour', label: 'Anti-social behaviour' },
|
value: 'Violence and sexual offences',
|
||||||
{ value: 'Criminal damage and arson', label: 'Criminal damage & arson' },
|
label: 'Violence & sexual offences',
|
||||||
{ value: 'Public order', label: 'Public order' },
|
labelKey: 'crimeTypes.violenceAndSexualOffences',
|
||||||
{ value: 'Shoplifting', label: 'Shoplifting' },
|
},
|
||||||
{ value: 'Vehicle crime', label: 'Vehicle crime' },
|
{
|
||||||
{ value: 'Burglary', label: 'Burglary' },
|
value: 'Anti-social behaviour',
|
||||||
{ value: 'Other theft', label: 'Other theft' },
|
label: 'Anti-social behaviour',
|
||||||
{ value: 'Theft from the person', label: 'Theft from the person' },
|
labelKey: 'crimeTypes.antiSocialBehaviour',
|
||||||
{ value: 'Bicycle theft', label: 'Bicycle theft' },
|
},
|
||||||
{ value: 'Drugs', label: 'Drugs' },
|
{
|
||||||
{ value: 'Robbery', label: 'Robbery' },
|
value: 'Criminal damage and arson',
|
||||||
{ value: 'Possession of weapons', label: 'Possession of weapons' },
|
label: 'Criminal damage & arson',
|
||||||
{ value: 'Other crime', label: 'Other crime' },
|
labelKey: 'crimeTypes.criminalDamageAndArson',
|
||||||
|
},
|
||||||
|
{ value: 'Public order', label: 'Public order', labelKey: 'crimeTypes.publicOrder' },
|
||||||
|
{ value: 'Shoplifting', label: 'Shoplifting', labelKey: 'crimeTypes.shoplifting' },
|
||||||
|
{ value: 'Vehicle crime', label: 'Vehicle crime', labelKey: 'crimeTypes.vehicleCrime' },
|
||||||
|
{ value: 'Burglary', label: 'Burglary', labelKey: 'crimeTypes.burglary' },
|
||||||
|
{ value: 'Other theft', label: 'Other theft', labelKey: 'crimeTypes.otherTheft' },
|
||||||
|
{
|
||||||
|
value: 'Theft from the person',
|
||||||
|
label: 'Theft from the person',
|
||||||
|
labelKey: 'crimeTypes.theftFromThePerson',
|
||||||
|
},
|
||||||
|
{ value: 'Bicycle theft', label: 'Bicycle theft', labelKey: 'crimeTypes.bicycleTheft' },
|
||||||
|
{ value: 'Drugs', label: 'Drugs', labelKey: 'crimeTypes.drugs' },
|
||||||
|
{ value: 'Robbery', label: 'Robbery', labelKey: 'crimeTypes.robbery' },
|
||||||
|
{
|
||||||
|
value: 'Possession of weapons',
|
||||||
|
label: 'Possession of weapons',
|
||||||
|
labelKey: 'crimeTypes.possessionOfWeapons',
|
||||||
|
},
|
||||||
|
{ value: 'Other crime', label: 'Other crime', labelKey: 'crimeTypes.otherCrime' },
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value);
|
export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value);
|
||||||
|
|
|
||||||
|
|
@ -404,7 +404,12 @@ const FEATURE_ICON_PATHS: Record<string, ReactNode> = {
|
||||||
* Returns a complete SVG icon element for a given feature name, or null if unmapped.
|
* Returns a complete SVG icon element for a given feature name, or null if unmapped.
|
||||||
*/
|
*/
|
||||||
export function getFeatureIcon(featureName: string, className: string): ReactElement | null {
|
export function getFeatureIcon(featureName: string, className: string): ReactElement | null {
|
||||||
const paths = FEATURE_ICON_PATHS[featureName];
|
// Crime features ("Burglary (/yr, 7y)") share the bare type's legacy
|
||||||
|
// "(avg/yr)" icon key; both windows use the same icon.
|
||||||
|
const rateMatch = featureName.match(/^(.*) \(\/yr, \d+y\)$/);
|
||||||
|
const paths =
|
||||||
|
FEATURE_ICON_PATHS[featureName] ??
|
||||||
|
(rateMatch ? FEATURE_ICON_PATHS[`${rateMatch[1]} (avg/yr)`] : undefined);
|
||||||
if (!paths) return null;
|
if (!paths) return null;
|
||||||
return (
|
return (
|
||||||
<svg
|
<svg
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,16 @@
|
||||||
|
import type { FeatureFilters, FeatureMeta } from '../types';
|
||||||
|
import type { VariantFilterConfig } from './variant-filter';
|
||||||
|
|
||||||
|
export const QUALIFICATIONS_FILTER_NAME = 'Qualifications';
|
||||||
|
export const QUALIFICATIONS_FILTER_KEY_PREFIX = `${QUALIFICATIONS_FILTER_NAME}:`;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Census 2021 qualification-breakdown features (TS067). These render as a
|
* The Census 2021 "highest level of qualification" bands (TS067). They sum to
|
||||||
* single stacked "Qualifications" composition in the area pane (see
|
* 100% per neighbourhood and render as a single stacked "Qualifications"
|
||||||
* STACKED_GROUPS["Neighbours"] in consts.ts) and are display-only: they are
|
* composition in the area pane (see STACKED_GROUPS["Neighbours"] in consts.ts).
|
||||||
* hidden from the filter browser rather than offered as seven individual
|
* In the filter browser they fold into one "Qualifications" filter whose
|
||||||
* sliders, so the breakdown reads as a ratio without cluttering the filter list.
|
* dropdown selects a band — including "% Degree or higher" — rather than seven
|
||||||
|
* separate sliders.
|
||||||
*/
|
*/
|
||||||
export const QUALIFICATION_FEATURE_NAMES = [
|
export const QUALIFICATION_FEATURE_NAMES = [
|
||||||
'% No qualifications',
|
'% No qualifications',
|
||||||
|
|
@ -20,3 +27,103 @@ const QUALIFICATION_FEATURE_NAME_SET = new Set<string>(QUALIFICATION_FEATURE_NAM
|
||||||
export function isQualificationFeatureName(name: string): boolean {
|
export function isQualificationFeatureName(name: string): boolean {
|
||||||
return QUALIFICATION_FEATURE_NAME_SET.has(name);
|
return QUALIFICATION_FEATURE_NAME_SET.has(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isQualificationFilterName(name: string): boolean {
|
||||||
|
return isQualificationFeatureName(name) || name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createQualificationFilterKey(featureName: string, id: number | string): string {
|
||||||
|
return `${QUALIFICATIONS_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getQualificationFilterKeyId(name: string): string | null {
|
||||||
|
if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
|
||||||
|
const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
return lastColon === -1 ? null : rest.substring(lastColon + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseQualificationFilterKey(name: string): string | null {
|
||||||
|
if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
|
||||||
|
const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
if (lastColon === -1) return null;
|
||||||
|
|
||||||
|
const decoded = decodeURIComponent(rest.substring(0, lastColon));
|
||||||
|
return isQualificationFeatureName(decoded) ? decoded : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getQualificationFeatureName(name: string): string | null {
|
||||||
|
if (isQualificationFeatureName(name)) return name;
|
||||||
|
return parseQualificationFilterKey(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function replaceQualificationFilterKeySelection(key: string, featureName: string): string {
|
||||||
|
const id = getQualificationFilterKeyId(key) ?? '0';
|
||||||
|
return createQualificationFilterKey(featureName, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDefaultQualificationFeatureName(features: FeatureMeta[]): string | null {
|
||||||
|
return (
|
||||||
|
QUALIFICATION_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ??
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeQualificationFilters(filters: FeatureFilters): FeatureFilters {
|
||||||
|
let changed = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
|
||||||
|
for (const [name, value] of Object.entries(filters)) {
|
||||||
|
if (isQualificationFeatureName(name)) {
|
||||||
|
next[createQualificationFilterKey(name, Object.keys(next).length)] = value;
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[name] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed ? next : filters;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getQualificationFilterMeta(features: FeatureMeta[]): FeatureMeta {
|
||||||
|
const sourceFeatureName = getDefaultQualificationFeatureName(features);
|
||||||
|
const sourceFeature = sourceFeatureName
|
||||||
|
? features.find((feature) => feature.name === sourceFeatureName)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: QUALIFICATIONS_FILTER_NAME,
|
||||||
|
type: 'numeric',
|
||||||
|
group: 'Neighbours',
|
||||||
|
min: sourceFeature?.min ?? 0,
|
||||||
|
max: sourceFeature?.max ?? 100,
|
||||||
|
step: 1,
|
||||||
|
description:
|
||||||
|
'Share of residents (16+) by highest qualification, from no qualifications to a degree or higher',
|
||||||
|
detail:
|
||||||
|
'Filter by one Census 2021 (TS067) highest-qualification band at a time, e.g. the percentage of residents whose highest qualification is a degree or higher.',
|
||||||
|
source: 'census-2021',
|
||||||
|
suffix: '%',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clampQualificationRange(
|
||||||
|
value: [number, number],
|
||||||
|
feature?: FeatureMeta
|
||||||
|
): [number, number] {
|
||||||
|
const min = feature?.histogram?.min ?? feature?.min ?? 0;
|
||||||
|
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
|
||||||
|
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
|
||||||
|
}
|
||||||
|
|
||||||
|
export const QUALIFICATION_VARIANT_CONFIG: VariantFilterConfig = {
|
||||||
|
filterName: QUALIFICATIONS_FILTER_NAME,
|
||||||
|
featureNames: QUALIFICATION_FEATURE_NAMES,
|
||||||
|
dropdownLabelKey: 'filters.qualificationLevel',
|
||||||
|
getFilterMeta: getQualificationFilterMeta,
|
||||||
|
getDefaultFeatureName: getDefaultQualificationFeatureName,
|
||||||
|
getFeatureName: getQualificationFeatureName,
|
||||||
|
replaceFilterKeySelection: replaceQualificationFilterKeySelection,
|
||||||
|
clampRange: clampQualificationRange,
|
||||||
|
};
|
||||||
|
|
|
||||||
121
frontend/src/lib/tenure-filter.ts
Normal file
121
frontend/src/lib/tenure-filter.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
import type { FeatureFilters, FeatureMeta } from '../types';
|
||||||
|
import type { VariantFilterConfig } from './variant-filter';
|
||||||
|
|
||||||
|
export const TENURE_FILTER_NAME = 'Tenure';
|
||||||
|
export const TENURE_FILTER_KEY_PREFIX = `${TENURE_FILTER_NAME}:`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Census 2021 housing tenure categories (TS054). They sum to 100% per
|
||||||
|
* neighbourhood and render as a single stacked "Tenure" composition in the area
|
||||||
|
* pane (see STACKED_GROUPS["Neighbours"] in consts.ts). In the filter browser
|
||||||
|
* they fold into one "Tenure" filter whose dropdown selects a category —
|
||||||
|
* owner-occupied, social rent or private rent — rather than three separate
|
||||||
|
* sliders.
|
||||||
|
*/
|
||||||
|
export const TENURE_FEATURE_NAMES = [
|
||||||
|
'% Owner occupied',
|
||||||
|
'% Social rent',
|
||||||
|
'% Private rent',
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const TENURE_FEATURE_NAME_SET = new Set<string>(TENURE_FEATURE_NAMES);
|
||||||
|
|
||||||
|
export function isTenureFeatureName(name: string): boolean {
|
||||||
|
return TENURE_FEATURE_NAME_SET.has(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isTenureFilterName(name: string): boolean {
|
||||||
|
return isTenureFeatureName(name) || name.startsWith(TENURE_FILTER_KEY_PREFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createTenureFilterKey(featureName: string, id: number | string): string {
|
||||||
|
return `${TENURE_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getTenureFilterKeyId(name: string): string | null {
|
||||||
|
if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
|
||||||
|
const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
return lastColon === -1 ? null : rest.substring(lastColon + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseTenureFilterKey(name: string): string | null {
|
||||||
|
if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
|
||||||
|
const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
|
||||||
|
const lastColon = rest.lastIndexOf(':');
|
||||||
|
if (lastColon === -1) return null;
|
||||||
|
|
||||||
|
const decoded = decodeURIComponent(rest.substring(0, lastColon));
|
||||||
|
return isTenureFeatureName(decoded) ? decoded : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getTenureFeatureName(name: string): string | null {
|
||||||
|
if (isTenureFeatureName(name)) return name;
|
||||||
|
return parseTenureFilterKey(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function replaceTenureFilterKeySelection(key: string, featureName: string): string {
|
||||||
|
const id = getTenureFilterKeyId(key) ?? '0';
|
||||||
|
return createTenureFilterKey(featureName, id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDefaultTenureFeatureName(features: FeatureMeta[]): string | null {
|
||||||
|
return (
|
||||||
|
TENURE_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ?? null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeTenureFilters(filters: FeatureFilters): FeatureFilters {
|
||||||
|
let changed = false;
|
||||||
|
const next: FeatureFilters = {};
|
||||||
|
|
||||||
|
for (const [name, value] of Object.entries(filters)) {
|
||||||
|
if (isTenureFeatureName(name)) {
|
||||||
|
next[createTenureFilterKey(name, Object.keys(next).length)] = value;
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
next[name] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed ? next : filters;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getTenureFilterMeta(features: FeatureMeta[]): FeatureMeta {
|
||||||
|
const sourceFeatureName = getDefaultTenureFeatureName(features);
|
||||||
|
const sourceFeature = sourceFeatureName
|
||||||
|
? features.find((feature) => feature.name === sourceFeatureName)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: TENURE_FILTER_NAME,
|
||||||
|
type: 'numeric',
|
||||||
|
group: 'Neighbours',
|
||||||
|
min: sourceFeature?.min ?? 0,
|
||||||
|
max: sourceFeature?.max ?? 100,
|
||||||
|
step: 1,
|
||||||
|
description:
|
||||||
|
'Share of households that own their home, rent from a social landlord, or rent privately',
|
||||||
|
detail:
|
||||||
|
'Filter by one Census 2021 (TS054) housing tenure category at a time, e.g. the percentage of households that own their home.',
|
||||||
|
source: 'census-2021',
|
||||||
|
suffix: '%',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clampTenureRange(value: [number, number], feature?: FeatureMeta): [number, number] {
|
||||||
|
const min = feature?.histogram?.min ?? feature?.min ?? 0;
|
||||||
|
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
|
||||||
|
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
|
||||||
|
}
|
||||||
|
|
||||||
|
export const TENURE_VARIANT_CONFIG: VariantFilterConfig = {
|
||||||
|
filterName: TENURE_FILTER_NAME,
|
||||||
|
featureNames: TENURE_FEATURE_NAMES,
|
||||||
|
dropdownLabelKey: 'filters.tenureType',
|
||||||
|
getFilterMeta: getTenureFilterMeta,
|
||||||
|
getDefaultFeatureName: getDefaultTenureFeatureName,
|
||||||
|
getFeatureName: getTenureFeatureName,
|
||||||
|
replaceFilterKeySelection: replaceTenureFilterKeySelection,
|
||||||
|
clampRange: clampTenureRange,
|
||||||
|
};
|
||||||
|
|
@ -5,8 +5,11 @@ import { parseUrlState, stateToParams } from './url-state';
|
||||||
import { INITIAL_VIEW_STATE } from './consts';
|
import { INITIAL_VIEW_STATE } from './consts';
|
||||||
import { createSchoolFilterKey } from './school-filter';
|
import { createSchoolFilterKey } from './school-filter';
|
||||||
import { createSpecificCrimeFilterKey } from './crime-filter';
|
import { createSpecificCrimeFilterKey } from './crime-filter';
|
||||||
|
import { createCrimeSeverityFilterKey } from './crime-severity-filter';
|
||||||
import { createElectionVoteShareFilterKey } from './election-filter';
|
import { createElectionVoteShareFilterKey } from './election-filter';
|
||||||
import { createEthnicityFilterKey } from './ethnicity-filter';
|
import { createEthnicityFilterKey } from './ethnicity-filter';
|
||||||
|
import { createQualificationFilterKey } from './qualification-filter';
|
||||||
|
import { createTenureFilterKey } from './tenure-filter';
|
||||||
import {
|
import {
|
||||||
POI_COUNT_2KM_FILTER_NAME,
|
POI_COUNT_2KM_FILTER_NAME,
|
||||||
TRANSPORT_DISTANCE_FILTER_NAME,
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
|
@ -405,8 +408,8 @@ describe('url-state', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('round-trips repeated specific crime filters with dedicated URL params', () => {
|
it('round-trips repeated specific crime filters with dedicated URL params', () => {
|
||||||
const burglary = createSpecificCrimeFilterKey('Burglary (avg/yr)', 1);
|
const burglary = createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1);
|
||||||
const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2);
|
const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2);
|
||||||
|
|
||||||
const params = stateToParams(
|
const params = stateToParams(
|
||||||
null,
|
null,
|
||||||
|
|
@ -420,8 +423,8 @@ describe('url-state', () => {
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(params.getAll('crime')).toEqual([
|
expect(params.getAll('crime')).toEqual([
|
||||||
'Burglary (avg/yr):0:5',
|
'Burglary (/yr, 7y):0:5',
|
||||||
'Vehicle crime (avg/yr):1:10',
|
'Vehicle crime (/yr, 7y):1:10',
|
||||||
]);
|
]);
|
||||||
expect(params.getAll('filter')).toEqual([]);
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
|
@ -429,8 +432,42 @@ describe('url-state', () => {
|
||||||
const state = parseUrlState();
|
const state = parseUrlState();
|
||||||
|
|
||||||
expect(state.filters).toEqual({
|
expect(state.filters).toEqual({
|
||||||
[createSpecificCrimeFilterKey('Burglary (avg/yr)', 0)]: [0, 5],
|
[createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 0)]: [0, 5],
|
||||||
[createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 1)]: [1, 10],
|
[createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 1)]: [1, 10],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('round-trips serious/minor crime severity filters with dedicated URL params', () => {
|
||||||
|
const serious = createCrimeSeverityFilterKey(
|
||||||
|
'Serious crime',
|
||||||
|
'Serious crime (/yr, 7y)',
|
||||||
|
0
|
||||||
|
);
|
||||||
|
const minor = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1);
|
||||||
|
|
||||||
|
const params = stateToParams(
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
[serious]: [0, 12],
|
||||||
|
[minor]: [3, 40],
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
new Set(),
|
||||||
|
'area'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(params.getAll('crimeSeverity')).toEqual([
|
||||||
|
'Serious crime (/yr, 7y):0:12',
|
||||||
|
'Minor crime (/yr, 2y):3:40',
|
||||||
|
]);
|
||||||
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
window.history.replaceState({}, '', `/?${params.toString()}`);
|
||||||
|
const state = parseUrlState();
|
||||||
|
|
||||||
|
expect(state.filters).toEqual({
|
||||||
|
[createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0)]: [0, 12],
|
||||||
|
[createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1)]: [3, 40],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -488,6 +525,65 @@ describe('url-state', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('round-trips repeated qualification filters with dedicated URL params', () => {
|
||||||
|
// "% Degree or higher" exercises the leading-`%` + space encoding path.
|
||||||
|
const degree = createQualificationFilterKey('% Degree or higher', 1);
|
||||||
|
const noQuals = createQualificationFilterKey('% No qualifications', 2);
|
||||||
|
|
||||||
|
const params = stateToParams(
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
[degree]: [20, 60],
|
||||||
|
[noQuals]: [0, 25],
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
new Set(),
|
||||||
|
'area'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(params.getAll('qualification')).toEqual([
|
||||||
|
'% Degree or higher:20:60',
|
||||||
|
'% No qualifications:0:25',
|
||||||
|
]);
|
||||||
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
window.history.replaceState({}, '', `/?${params.toString()}`);
|
||||||
|
const state = parseUrlState();
|
||||||
|
|
||||||
|
expect(state.filters).toEqual({
|
||||||
|
[createQualificationFilterKey('% Degree or higher', 0)]: [20, 60],
|
||||||
|
[createQualificationFilterKey('% No qualifications', 1)]: [0, 25],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('round-trips repeated tenure filters with dedicated URL params', () => {
|
||||||
|
// "% Owner occupied" exercises the leading-`%` + space encoding path.
|
||||||
|
const owner = createTenureFilterKey('% Owner occupied', 1);
|
||||||
|
const privateRent = createTenureFilterKey('% Private rent', 2);
|
||||||
|
|
||||||
|
const params = stateToParams(
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
[owner]: [20, 60],
|
||||||
|
[privateRent]: [0, 25],
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
new Set(),
|
||||||
|
'area'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(params.getAll('tenure')).toEqual(['% Owner occupied:20:60', '% Private rent:0:25']);
|
||||||
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
window.history.replaceState({}, '', `/?${params.toString()}`);
|
||||||
|
const state = parseUrlState();
|
||||||
|
|
||||||
|
expect(state.filters).toEqual({
|
||||||
|
[createTenureFilterKey('% Owner occupied', 0)]: [20, 60],
|
||||||
|
[createTenureFilterKey('% Private rent', 1)]: [0, 25],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('round-trips repeated amenity distance filters with dedicated URL params', () => {
|
it('round-trips repeated amenity distance filters with dedicated URL params', () => {
|
||||||
const park = createPoiDistanceFilterKey('Distance to nearest amenity (Park) (km)', 3);
|
const park = createPoiDistanceFilterKey('Distance to nearest amenity (Park) (km)', 3);
|
||||||
const cafe = createPoiDistanceFilterKey('Distance to nearest amenity (Café) (km)', 4);
|
const cafe = createPoiDistanceFilterKey('Distance to nearest amenity (Café) (km)', 4);
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,13 @@ import {
|
||||||
isSpecificCrimeFeatureName,
|
isSpecificCrimeFeatureName,
|
||||||
isSpecificCrimeFilterName,
|
isSpecificCrimeFilterName,
|
||||||
} from './crime-filter';
|
} from './crime-filter';
|
||||||
|
import {
|
||||||
|
createCrimeSeverityFilterKey,
|
||||||
|
getCrimeSeverityFeatureName,
|
||||||
|
getCrimeSeverityFilterName,
|
||||||
|
isCrimeSeverityFeatureName,
|
||||||
|
isCrimeSeverityFilterName,
|
||||||
|
} from './crime-severity-filter';
|
||||||
import {
|
import {
|
||||||
ELECTION_VOTE_SHARE_FILTER_NAME,
|
ELECTION_VOTE_SHARE_FILTER_NAME,
|
||||||
createElectionVoteShareFilterKey,
|
createElectionVoteShareFilterKey,
|
||||||
|
|
@ -36,6 +43,20 @@ import {
|
||||||
isEthnicityFeatureName,
|
isEthnicityFeatureName,
|
||||||
isEthnicityFilterName,
|
isEthnicityFilterName,
|
||||||
} from './ethnicity-filter';
|
} from './ethnicity-filter';
|
||||||
|
import {
|
||||||
|
QUALIFICATIONS_FILTER_NAME,
|
||||||
|
createQualificationFilterKey,
|
||||||
|
getQualificationFeatureName,
|
||||||
|
isQualificationFeatureName,
|
||||||
|
isQualificationFilterName,
|
||||||
|
} from './qualification-filter';
|
||||||
|
import {
|
||||||
|
TENURE_FILTER_NAME,
|
||||||
|
createTenureFilterKey,
|
||||||
|
getTenureFeatureName,
|
||||||
|
isTenureFeatureName,
|
||||||
|
isTenureFilterName,
|
||||||
|
} from './tenure-filter';
|
||||||
import {
|
import {
|
||||||
POI_DISTANCE_FILTER_NAME,
|
POI_DISTANCE_FILTER_NAME,
|
||||||
TRANSPORT_DISTANCE_FILTER_NAME,
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
|
@ -83,8 +104,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
const filterParams = params.getAll('filter');
|
const filterParams = params.getAll('filter');
|
||||||
const schoolParams = params.getAll('school');
|
const schoolParams = params.getAll('school');
|
||||||
const crimeParams = params.getAll('crime');
|
const crimeParams = params.getAll('crime');
|
||||||
|
const crimeSeverityParams = params.getAll('crimeSeverity');
|
||||||
const voteShareParams = params.getAll('voteShare');
|
const voteShareParams = params.getAll('voteShare');
|
||||||
const ethnicityParams = params.getAll('ethnicity');
|
const ethnicityParams = params.getAll('ethnicity');
|
||||||
|
const qualificationParams = params.getAll('qualification');
|
||||||
|
const tenureParams = params.getAll('tenure');
|
||||||
const amenityDistanceParams = params.getAll('amenityDistance');
|
const amenityDistanceParams = params.getAll('amenityDistance');
|
||||||
const transportDistanceParams = params.getAll('transportDistance');
|
const transportDistanceParams = params.getAll('transportDistance');
|
||||||
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
||||||
|
|
@ -93,8 +117,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
filterParams.length === 0 &&
|
filterParams.length === 0 &&
|
||||||
schoolParams.length === 0 &&
|
schoolParams.length === 0 &&
|
||||||
crimeParams.length === 0 &&
|
crimeParams.length === 0 &&
|
||||||
|
crimeSeverityParams.length === 0 &&
|
||||||
voteShareParams.length === 0 &&
|
voteShareParams.length === 0 &&
|
||||||
ethnicityParams.length === 0 &&
|
ethnicityParams.length === 0 &&
|
||||||
|
qualificationParams.length === 0 &&
|
||||||
|
tenureParams.length === 0 &&
|
||||||
amenityDistanceParams.length === 0 &&
|
amenityDistanceParams.length === 0 &&
|
||||||
transportDistanceParams.length === 0 &&
|
transportDistanceParams.length === 0 &&
|
||||||
amenityCount2KmParams.length === 0 &&
|
amenityCount2KmParams.length === 0 &&
|
||||||
|
|
@ -155,6 +182,19 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
filters[createSpecificCrimeFilterKey(featureName, index)] = [min, max];
|
filters[createSpecificCrimeFilterKey(featureName, index)] = [min, max];
|
||||||
});
|
});
|
||||||
|
|
||||||
|
crimeSeverityParams.forEach((entry, index) => {
|
||||||
|
const parts = entry.split(':');
|
||||||
|
if (parts.length < 3) return;
|
||||||
|
const featureName = parts.slice(0, -2).join(':');
|
||||||
|
const min = Number(parts[parts.length - 2]);
|
||||||
|
const max = Number(parts[parts.length - 1]);
|
||||||
|
const filterName = getCrimeSeverityFilterName(featureName);
|
||||||
|
if (!isCrimeSeverityFeatureName(featureName) || !filterName || isNaN(min) || isNaN(max)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
filters[createCrimeSeverityFilterKey(filterName, featureName, index)] = [min, max];
|
||||||
|
});
|
||||||
|
|
||||||
voteShareParams.forEach((entry, index) => {
|
voteShareParams.forEach((entry, index) => {
|
||||||
const parts = entry.split(':');
|
const parts = entry.split(':');
|
||||||
if (parts.length < 3) return;
|
if (parts.length < 3) return;
|
||||||
|
|
@ -179,6 +219,30 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
filters[createEthnicityFilterKey(featureName, index)] = [min, max];
|
filters[createEthnicityFilterKey(featureName, index)] = [min, max];
|
||||||
});
|
});
|
||||||
|
|
||||||
|
qualificationParams.forEach((entry, index) => {
|
||||||
|
const parts = entry.split(':');
|
||||||
|
if (parts.length < 3) return;
|
||||||
|
const featureName = parts.slice(0, -2).join(':');
|
||||||
|
const min = Number(parts[parts.length - 2]);
|
||||||
|
const max = Number(parts[parts.length - 1]);
|
||||||
|
if (!isQualificationFeatureName(featureName) || isNaN(min) || isNaN(max)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
filters[createQualificationFilterKey(featureName, index)] = [min, max];
|
||||||
|
});
|
||||||
|
|
||||||
|
tenureParams.forEach((entry, index) => {
|
||||||
|
const parts = entry.split(':');
|
||||||
|
if (parts.length < 3) return;
|
||||||
|
const featureName = parts.slice(0, -2).join(':');
|
||||||
|
const min = Number(parts[parts.length - 2]);
|
||||||
|
const max = Number(parts[parts.length - 1]);
|
||||||
|
if (!isTenureFeatureName(featureName) || isNaN(min) || isNaN(max)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
filters[createTenureFilterKey(featureName, index)] = [min, max];
|
||||||
|
});
|
||||||
|
|
||||||
const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => {
|
const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => {
|
||||||
entries.forEach((entry, index) => {
|
entries.forEach((entry, index) => {
|
||||||
const parts = entry.split(':');
|
const parts = entry.split(':');
|
||||||
|
|
@ -401,6 +465,13 @@ export function stateToParams(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const crimeSeverityFeatureName = getCrimeSeverityFeatureName(name);
|
||||||
|
if (crimeSeverityFeatureName && isCrimeSeverityFilterName(name)) {
|
||||||
|
const [min, max] = value as [number, number];
|
||||||
|
params.append('crimeSeverity', `${crimeSeverityFeatureName}:${min}:${max}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
|
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
|
||||||
if (electionVoteShareFeatureName && isElectionVoteShareFilterName(name)) {
|
if (electionVoteShareFeatureName && isElectionVoteShareFilterName(name)) {
|
||||||
const [min, max] = value as [number, number];
|
const [min, max] = value as [number, number];
|
||||||
|
|
@ -415,6 +486,20 @@ export function stateToParams(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const qualificationFeatureName = getQualificationFeatureName(name);
|
||||||
|
if (qualificationFeatureName && isQualificationFilterName(name)) {
|
||||||
|
const [min, max] = value as [number, number];
|
||||||
|
params.append('qualification', `${qualificationFeatureName}:${min}:${max}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const tenureFeatureName = getTenureFeatureName(name);
|
||||||
|
if (tenureFeatureName && isTenureFilterName(name)) {
|
||||||
|
const [min, max] = value as [number, number];
|
||||||
|
params.append('tenure', `${tenureFeatureName}:${min}:${max}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const amenityDistanceFeatureName = getPoiDistanceFeatureName(name);
|
const amenityDistanceFeatureName = getPoiDistanceFeatureName(name);
|
||||||
if (amenityDistanceFeatureName && isPoiDistanceFilterName(name)) {
|
if (amenityDistanceFeatureName && isPoiDistanceFilterName(name)) {
|
||||||
const [min, max] = value as [number, number];
|
const [min, max] = value as [number, number];
|
||||||
|
|
@ -522,8 +607,11 @@ export function summarizeParams(queryString: string): string {
|
||||||
const filterParams = params.getAll('filter');
|
const filterParams = params.getAll('filter');
|
||||||
const schoolParams = params.getAll('school');
|
const schoolParams = params.getAll('school');
|
||||||
const crimeParams = params.getAll('crime');
|
const crimeParams = params.getAll('crime');
|
||||||
|
const crimeSeverityParams = params.getAll('crimeSeverity');
|
||||||
const voteShareParams = params.getAll('voteShare');
|
const voteShareParams = params.getAll('voteShare');
|
||||||
const ethnicityParams = params.getAll('ethnicity');
|
const ethnicityParams = params.getAll('ethnicity');
|
||||||
|
const qualificationParams = params.getAll('qualification');
|
||||||
|
const tenureParams = params.getAll('tenure');
|
||||||
const amenityDistanceParams = params.getAll('amenityDistance');
|
const amenityDistanceParams = params.getAll('amenityDistance');
|
||||||
const transportDistanceParams = params.getAll('transportDistance');
|
const transportDistanceParams = params.getAll('transportDistance');
|
||||||
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
||||||
|
|
@ -532,8 +620,11 @@ export function summarizeParams(queryString: string): string {
|
||||||
filterParams.length > 0 ||
|
filterParams.length > 0 ||
|
||||||
schoolParams.length > 0 ||
|
schoolParams.length > 0 ||
|
||||||
crimeParams.length > 0 ||
|
crimeParams.length > 0 ||
|
||||||
|
crimeSeverityParams.length > 0 ||
|
||||||
voteShareParams.length > 0 ||
|
voteShareParams.length > 0 ||
|
||||||
ethnicityParams.length > 0 ||
|
ethnicityParams.length > 0 ||
|
||||||
|
qualificationParams.length > 0 ||
|
||||||
|
tenureParams.length > 0 ||
|
||||||
amenityDistanceParams.length > 0 ||
|
amenityDistanceParams.length > 0 ||
|
||||||
transportDistanceParams.length > 0 ||
|
transportDistanceParams.length > 0 ||
|
||||||
amenityCount2KmParams.length > 0 ||
|
amenityCount2KmParams.length > 0 ||
|
||||||
|
|
@ -544,8 +635,11 @@ export function summarizeParams(queryString: string): string {
|
||||||
const colonIdx = entry.indexOf(':');
|
const colonIdx = entry.indexOf(':');
|
||||||
const name = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
|
const name = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
|
||||||
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
|
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
|
||||||
|
if (isCrimeSeverityFeatureName(name)) return getCrimeSeverityFilterName(name) ?? name;
|
||||||
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
|
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
|
||||||
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
|
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
|
||||||
|
if (isQualificationFeatureName(name)) return QUALIFICATIONS_FILTER_NAME;
|
||||||
|
if (isTenureFeatureName(name)) return TENURE_FILTER_NAME;
|
||||||
const poiFilterName = getPoiFilterName(name);
|
const poiFilterName = getPoiFilterName(name);
|
||||||
if (poiFilterName) return poiFilterName;
|
if (poiFilterName) return poiFilterName;
|
||||||
return name;
|
return name;
|
||||||
|
|
@ -555,12 +649,24 @@ export function summarizeParams(queryString: string): string {
|
||||||
for (let i = 0; i < crimeParams.length; i++) {
|
for (let i = 0; i < crimeParams.length; i++) {
|
||||||
filterNames.push(SPECIFIC_CRIMES_FILTER_NAME);
|
filterNames.push(SPECIFIC_CRIMES_FILTER_NAME);
|
||||||
}
|
}
|
||||||
|
for (const entry of crimeSeverityParams) {
|
||||||
|
const colonIdx = entry.indexOf(':');
|
||||||
|
const featureName = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
|
||||||
|
const severityFilterName = getCrimeSeverityFilterName(featureName);
|
||||||
|
if (severityFilterName) filterNames.push(severityFilterName);
|
||||||
|
}
|
||||||
for (let i = 0; i < voteShareParams.length; i++) {
|
for (let i = 0; i < voteShareParams.length; i++) {
|
||||||
filterNames.push(ELECTION_VOTE_SHARE_FILTER_NAME);
|
filterNames.push(ELECTION_VOTE_SHARE_FILTER_NAME);
|
||||||
}
|
}
|
||||||
for (let i = 0; i < ethnicityParams.length; i++) {
|
for (let i = 0; i < ethnicityParams.length; i++) {
|
||||||
filterNames.push(ETHNICITIES_FILTER_NAME);
|
filterNames.push(ETHNICITIES_FILTER_NAME);
|
||||||
}
|
}
|
||||||
|
for (let i = 0; i < qualificationParams.length; i++) {
|
||||||
|
filterNames.push(QUALIFICATIONS_FILTER_NAME);
|
||||||
|
}
|
||||||
|
for (let i = 0; i < tenureParams.length; i++) {
|
||||||
|
filterNames.push(TENURE_FILTER_NAME);
|
||||||
|
}
|
||||||
for (let i = 0; i < amenityDistanceParams.length; i++) {
|
for (let i = 0; i < amenityDistanceParams.length; i++) {
|
||||||
filterNames.push(POI_DISTANCE_FILTER_NAME);
|
filterNames.push(POI_DISTANCE_FILTER_NAME);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
80
frontend/src/lib/variant-filter.ts
Normal file
80
frontend/src/lib/variant-filter.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
import type { FeatureMeta } from '../types';
|
||||||
|
|
||||||
|
/** i18n keys (typed for the strict `t()`) usable as a variant dropdown label. */
|
||||||
|
type VariantDropdownLabelKey =
|
||||||
|
| 'filters.crimeType'
|
||||||
|
| 'filters.qualificationLevel'
|
||||||
|
| 'filters.tenureType';
|
||||||
|
|
||||||
|
/** i18n keys (typed for the strict `t()`) usable as a window-toggle label. */
|
||||||
|
type VariantWindowLabelKey =
|
||||||
|
| 'filters.crimeWindow'
|
||||||
|
| 'filters.crimeWindow7y'
|
||||||
|
| 'filters.crimeWindow2y';
|
||||||
|
|
||||||
|
/** One option in a variant filter's secondary window/period toggle. */
|
||||||
|
export interface VariantWindowOption {
|
||||||
|
/** Stable id encoded inside the feature name (e.g. "7y"). */
|
||||||
|
id: string;
|
||||||
|
/** i18n key for the toggle button label. */
|
||||||
|
labelKey: VariantWindowLabelKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optional secondary axis for a variant filter: the same variant measured over
|
||||||
|
* a different time window (e.g. crime rates over 7 vs 2 years). The dropdown
|
||||||
|
* still picks the variant; this toggle picks the window. Both ultimately select
|
||||||
|
* a single backend feature name, so switching either re-points the filter key.
|
||||||
|
*/
|
||||||
|
export interface VariantWindowConfig {
|
||||||
|
/** Toggle options in display order. */
|
||||||
|
options: VariantWindowOption[];
|
||||||
|
/** Optional i18n key for a small label above the toggle. */
|
||||||
|
labelKey?: VariantWindowLabelKey;
|
||||||
|
/** Window id of a feature name (e.g. "7y"), or null if unrecognized. */
|
||||||
|
getWindow: (featureName: string) => string | null;
|
||||||
|
/** The same variant's feature name in a different window. */
|
||||||
|
withWindow: (featureName: string, windowId: string) => string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared shape for the "pick one backend feature variant, filter its range"
|
||||||
|
* family of client-side aggregate filters. Several Census/police breakdowns
|
||||||
|
* (specific crimes, qualifications, …) are dozens of individual percentage
|
||||||
|
* features that would clutter the filter browser as separate sliders. Instead
|
||||||
|
* each is folded into a single named filter whose card carries a dropdown to
|
||||||
|
* choose the variant, reusing one component ([`VariantFilterCard`]).
|
||||||
|
*
|
||||||
|
* Each filter library (e.g. `crime-filter`, `qualification-filter`) exports a
|
||||||
|
* config so the card stays variant-agnostic.
|
||||||
|
*/
|
||||||
|
export interface VariantFilterConfig {
|
||||||
|
/** Display name of the aggregate filter, e.g. "Specific crimes". */
|
||||||
|
filterName: string;
|
||||||
|
/**
|
||||||
|
* Backend feature names enumerating the dropdown options, in display order.
|
||||||
|
* With a [`window`] config these are the canonical (default-window) names;
|
||||||
|
* the card re-points each to the currently selected window.
|
||||||
|
*/
|
||||||
|
featureNames: readonly string[];
|
||||||
|
/** i18n key for the dropdown label, e.g. "filters.crimeType". */
|
||||||
|
dropdownLabelKey: VariantDropdownLabelKey;
|
||||||
|
/** Synthetic [`FeatureMeta`] used for the aggregate filter's own label/info. */
|
||||||
|
getFilterMeta: (features: FeatureMeta[]) => FeatureMeta;
|
||||||
|
/** First selectable variant present in `features`, or null if none. */
|
||||||
|
getDefaultFeatureName: (features: FeatureMeta[]) => string | null;
|
||||||
|
/** Backend feature name for a filter key (or a bare feature name). */
|
||||||
|
getFeatureName: (name: string) => string | null;
|
||||||
|
/** Rewrite a filter key to point at a different variant, keeping its id. */
|
||||||
|
replaceFilterKeySelection: (key: string, featureName: string) => string;
|
||||||
|
/** Clamp a [min, max] range into the selected feature's bounds. */
|
||||||
|
clampRange: (value: [number, number], feature?: FeatureMeta) => [number, number];
|
||||||
|
/**
|
||||||
|
* Server-translatable source value for a dropdown option's label; the card
|
||||||
|
* renders `ts(...)` of it. Defaults to the feature name itself. Useful with a
|
||||||
|
* [`window`] toggle so the label can be the bare variant (no window suffix).
|
||||||
|
*/
|
||||||
|
getOptionLabelSource?: (featureName: string) => string;
|
||||||
|
/** Optional secondary window/period toggle (e.g. 7- vs 2-year crime rates). */
|
||||||
|
window?: VariantWindowConfig;
|
||||||
|
}
|
||||||
|
|
@ -316,26 +316,17 @@ export interface CrimeYearStats {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CrimeAreaAverage {
|
export interface CrimeAreaAverage {
|
||||||
/** Full rate-feature name (e.g. "Burglary (per 1k/yr, 7y)"). */
|
/** Full crime-feature name (e.g. "Burglary (/yr, 7y)"). */
|
||||||
name: string;
|
name: string;
|
||||||
/** Exact national mean rate. Preferred over the histogram-bin national
|
/** Exact national mean count. Preferred over the histogram-bin national
|
||||||
* average for crime so all reference numbers share one estimator. */
|
* average for crime so all reference numbers share one estimator. */
|
||||||
national?: number;
|
national?: number;
|
||||||
/** Mean rate across the selection's outcode. */
|
/** Mean count across the selection's outcode. */
|
||||||
outcode?: number;
|
outcode?: number;
|
||||||
/** Mean rate across the selection's postcode sector. */
|
/** Mean count across the selection's postcode sector. */
|
||||||
sector?: number;
|
sector?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CrimeRawStats {
|
|
||||||
/** Bare crime type (e.g. "Burglary"). */
|
|
||||||
name: string;
|
|
||||||
/** Mean recorded incidents/yr over the last 7 years. */
|
|
||||||
per_yr_7y?: number;
|
|
||||||
/** Mean recorded incidents/yr over the last 2 years. */
|
|
||||||
per_yr_2y?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** One individual police.uk crime, from /api/crime-records. */
|
/** One individual police.uk crime, from /api/crime-records. */
|
||||||
export interface CrimeIncident {
|
export interface CrimeIncident {
|
||||||
/** "YYYY-MM". */
|
/** "YYYY-MM". */
|
||||||
|
|
@ -352,6 +343,11 @@ export interface CrimeRecordsResponse {
|
||||||
records: CrimeIncident[];
|
records: CrimeIncident[];
|
||||||
total: number;
|
total: number;
|
||||||
offset: number;
|
offset: number;
|
||||||
|
/**
|
||||||
|
* Server flag meaning "more pages exist beyond this one". The client derives
|
||||||
|
* pagination from `total` vs `records.length`, so this is currently not
|
||||||
|
* surfaced in the UI (kept to mirror the server response shape).
|
||||||
|
*/
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -386,12 +382,9 @@ export interface HexagonStatsResponse {
|
||||||
/** Postcode sector (e.g. "E14 2") of the selection's central postcode, when
|
/** Postcode sector (e.g. "E14 2") of the selection's central postcode, when
|
||||||
* sector crime averages are available for it. */
|
* sector crime averages are available for it. */
|
||||||
crime_sector?: string;
|
crime_sector?: string;
|
||||||
/** Per-rate-feature average rates across the central postcode's outcode and
|
/** Per-crime-feature average counts across the central postcode's outcode and
|
||||||
* sector, shown alongside the national average for each crime metric. */
|
* sector, shown alongside the national average for each crime metric. */
|
||||||
crime_area_averages?: CrimeAreaAverage[];
|
crime_area_averages?: CrimeAreaAverage[];
|
||||||
/** Raw (un-normalised) recorded incidents/yr per crime type, shown beside the
|
|
||||||
* normalised rate. Display-only. */
|
|
||||||
crime_raw?: CrimeRawStats[];
|
|
||||||
/** Total individual crime records (last 7 years) across the selection's
|
/** Total individual crime records (last 7 years) across the selection's
|
||||||
* postcodes — the count behind the "individual crimes" list. */
|
* postcodes — the count behind the "individual crimes" list. */
|
||||||
crime_total_records?: number;
|
crime_total_records?: number;
|
||||||
|
|
|
||||||
212
pipeline/transform/area_crime_averages.py
Normal file
212
pipeline/transform/area_crime_averages.py
Normal file
|
|
@ -0,0 +1,212 @@
|
||||||
|
"""Precompute per-outcode / per-postcode-sector / national mean headline crime
|
||||||
|
counts for the right pane's area comparison.
|
||||||
|
|
||||||
|
The right pane shows each crime metric next to its area context: the mean
|
||||||
|
average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
|
||||||
|
``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
|
||||||
|
within a postcode (the merge keys it on the postcode), so each postcode
|
||||||
|
contributes its single value weighted by how many properties sit in it — keeping
|
||||||
|
every scope on the same property-weighted basis as the per-selection mean, so the
|
||||||
|
four numbers (this selection / sector / outcode / nation) are directly
|
||||||
|
comparable. The national figure here is an EXACT property-weighted mean, which is
|
||||||
|
why it overrides the upward-biased histogram-bin national average for crime.
|
||||||
|
|
||||||
|
This used to be recomputed inside the server on every boot from the loaded
|
||||||
|
property matrix. It is a pure function of the two merge outputs, so it belongs in
|
||||||
|
the data build; the server now just loads the parquet this writes. Reading the
|
||||||
|
crime values from ``postcode.parquet`` and the per-postcode property weights from
|
||||||
|
``properties.parquet`` mirrors exactly the two inputs the server loads, so the
|
||||||
|
result matches what the server used to compute (minus its u16 quantization loss).
|
||||||
|
|
||||||
|
Output schema — one row per area:
|
||||||
|
|
||||||
|
scope : ``"national"`` | ``"outcode"`` | ``"sector"``
|
||||||
|
area : the outcode (``"E14"``) / sector (``"E14 2"``);
|
||||||
|
``""`` for the single national row
|
||||||
|
``<type> (/yr, 7y|2y)`` : Float32 property-weighted mean crime count per year
|
||||||
|
(null = the scope has no data for that type)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
# Filterable crime columns are the average-annual incident counts and carry this
|
||||||
|
# marker in postcode.parquet (e.g. "Burglary (/yr, 7y)"). We average those. The
|
||||||
|
# full column name is kept; the server discovers and keys area averages by the
|
||||||
|
# same names.
|
||||||
|
COUNT_MARKER = " (/yr, "
|
||||||
|
|
||||||
|
# `scope` discriminator values. The server's loader dispatches on these.
|
||||||
|
SCOPE_NATIONAL = "national"
|
||||||
|
SCOPE_OUTCODE = "outcode"
|
||||||
|
SCOPE_SECTOR = "sector"
|
||||||
|
|
||||||
|
# Area label on the national row — it spans the whole country, so it has no code.
|
||||||
|
NATIONAL_AREA = ""
|
||||||
|
|
||||||
|
# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
|
||||||
|
# "E14 2DG").
|
||||||
|
POSTCODE_COLUMN = "Postcode"
|
||||||
|
|
||||||
|
# Internal weight / split columns dropped before write.
|
||||||
|
_WEIGHT_COLUMN = "_weight"
|
||||||
|
_OUTCODE_COLUMN = "_outcode"
|
||||||
|
_SECTOR_COLUMN = "_sector"
|
||||||
|
|
||||||
|
|
||||||
|
def _crime_columns(columns: list[str]) -> list[str]:
|
||||||
|
crime_cols = [name for name in columns if COUNT_MARKER in name]
|
||||||
|
if not crime_cols:
|
||||||
|
raise ValueError(
|
||||||
|
f"postcode parquet has no '*{COUNT_MARKER}*' crime count columns to average"
|
||||||
|
)
|
||||||
|
return crime_cols
|
||||||
|
|
||||||
|
|
||||||
|
def _weighted_mean(column: str) -> pl.Expr:
|
||||||
|
"""Property-weighted mean of ``column`` that excludes nulls from BOTH the
|
||||||
|
value sum and the weight.
|
||||||
|
|
||||||
|
A null crime value is a genuine gap (the postcode's police force published no
|
||||||
|
usable data), not zero crime, so it must dilute neither the numerator nor the
|
||||||
|
denominator — exactly as the server's former estimator skipped NaN values.
|
||||||
|
Yields null when no postcode in the group has data for this type.
|
||||||
|
"""
|
||||||
|
weight = pl.col(_WEIGHT_COLUMN)
|
||||||
|
numerator = (pl.col(column) * weight).sum()
|
||||||
|
denominator = weight.filter(pl.col(column).is_not_null()).sum()
|
||||||
|
return (
|
||||||
|
pl.when(denominator > 0)
|
||||||
|
.then(numerator / denominator)
|
||||||
|
.otherwise(None)
|
||||||
|
.cast(pl.Float32)
|
||||||
|
.alias(column)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compute_area_crime_averages(
|
||||||
|
postcodes: pl.LazyFrame, properties: pl.LazyFrame
|
||||||
|
) -> pl.DataFrame:
|
||||||
|
"""Build the national / per-outcode / per-sector crime-average table.
|
||||||
|
|
||||||
|
``postcodes`` is the merge's postcode output (one row per active postcode,
|
||||||
|
carrying the ``"* (/yr, *)"`` crime count columns); ``properties`` is the merge's
|
||||||
|
per-property output, used only to weight each postcode by its property count.
|
||||||
|
"""
|
||||||
|
crime_cols = _crime_columns(postcodes.collect_schema().names())
|
||||||
|
|
||||||
|
# Property weight per postcode = how many property rows the server indexes
|
||||||
|
# under it. The inner join keeps only postcodes that actually carry
|
||||||
|
# properties, matching the server's per-postcode row index (a postcode with
|
||||||
|
# no properties never contributed to any average).
|
||||||
|
weights = properties.group_by(POSTCODE_COLUMN).agg(pl.len().alias(_WEIGHT_COLUMN))
|
||||||
|
|
||||||
|
# Outcode / sector of the spaced `pcds` postcode, matching the server's
|
||||||
|
# postcode_outcode / postcode_sector (split on the single space; sector =
|
||||||
|
# outcode + space + first inward character). Null where the form has no
|
||||||
|
# inward code, so such rows drop out of the per-area groups.
|
||||||
|
parts = pl.col(POSTCODE_COLUMN).str.splitn(" ", 2).struct
|
||||||
|
outward = parts.field("field_0")
|
||||||
|
inward = parts.field("field_1")
|
||||||
|
base = (
|
||||||
|
postcodes.select(POSTCODE_COLUMN, *crime_cols)
|
||||||
|
.join(weights, on=POSTCODE_COLUMN, how="inner")
|
||||||
|
.with_columns(
|
||||||
|
pl.when(inward.is_not_null())
|
||||||
|
.then(outward)
|
||||||
|
.otherwise(None)
|
||||||
|
.alias(_OUTCODE_COLUMN),
|
||||||
|
pl.when(inward.str.len_chars() >= 1)
|
||||||
|
.then(outward + pl.lit(" ") + inward.str.slice(0, 1))
|
||||||
|
.otherwise(None)
|
||||||
|
.alias(_SECTOR_COLUMN),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
mean_exprs = [_weighted_mean(column) for column in crime_cols]
|
||||||
|
|
||||||
|
national = base.select(
|
||||||
|
pl.lit(SCOPE_NATIONAL).alias("scope"),
|
||||||
|
pl.lit(NATIONAL_AREA).alias("area"),
|
||||||
|
*mean_exprs,
|
||||||
|
)
|
||||||
|
by_outcode = (
|
||||||
|
base.drop_nulls(_OUTCODE_COLUMN)
|
||||||
|
.group_by(_OUTCODE_COLUMN)
|
||||||
|
.agg(mean_exprs)
|
||||||
|
.select(
|
||||||
|
pl.lit(SCOPE_OUTCODE).alias("scope"),
|
||||||
|
pl.col(_OUTCODE_COLUMN).alias("area"),
|
||||||
|
*crime_cols,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
by_sector = (
|
||||||
|
base.drop_nulls(_SECTOR_COLUMN)
|
||||||
|
.group_by(_SECTOR_COLUMN)
|
||||||
|
.agg(mean_exprs)
|
||||||
|
.select(
|
||||||
|
pl.lit(SCOPE_SECTOR).alias("scope"),
|
||||||
|
pl.col(_SECTOR_COLUMN).alias("area"),
|
||||||
|
*crime_cols,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
result = pl.concat([national, by_outcode, by_sector], how="vertical").collect(
|
||||||
|
engine="streaming"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Drop per-area rows where every crime type is null: the server only created
|
||||||
|
# a map entry once a scope had at least one finite value, so an all-null
|
||||||
|
# outcode/sector reported no code at all. The national row is always kept (it
|
||||||
|
# always has data, and is emitted even for areas absent from both maps).
|
||||||
|
has_any = pl.any_horizontal(pl.col(column).is_not_null() for column in crime_cols)
|
||||||
|
return result.filter((pl.col("scope") == SCOPE_NATIONAL) | has_any)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=(
|
||||||
|
"Precompute national / per-outcode / per-sector mean headline crime "
|
||||||
|
"counts from the merge outputs"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--postcodes",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="postcode.parquet (area features, incl. the '* (/yr, *)' crime columns)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--properties",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="properties.parquet (per-property rows; supplies postcode property weights)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Output area_crime_averages.parquet path",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
result = compute_area_crime_averages(
|
||||||
|
pl.scan_parquet(args.postcodes), pl.scan_parquet(args.properties)
|
||||||
|
)
|
||||||
|
outcodes = result.filter(pl.col("scope") == SCOPE_OUTCODE).height
|
||||||
|
sectors = result.filter(pl.col("scope") == SCOPE_SECTOR).height
|
||||||
|
print(
|
||||||
|
f"Area crime averages: {result.height} rows "
|
||||||
|
f"({outcodes} outcodes, {sectors} sectors, "
|
||||||
|
f"{len(_crime_columns(result.columns))} crime types)"
|
||||||
|
)
|
||||||
|
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
result.write_parquet(args.output, compression="zstd")
|
||||||
|
print(f"Saved to {args.output}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -2,22 +2,26 @@
|
||||||
|
|
||||||
Instead of attributing each incident to its published LSOA code, this transform
|
Instead of attributing each incident to its published LSOA code, this transform
|
||||||
counts the anonymised incident *points* that fall within ``buffer_m`` (default
|
counts the anonymised incident *points* that fall within ``buffer_m`` (default
|
||||||
100m) of each postcode's boundary polygon (the polygon buffered outward). A point
|
50m) of each postcode's boundary polygon (the polygon buffered outward). A point
|
||||||
inside several overlapping buffers counts for each postcode -- the same
|
inside several overlapping buffers counts for each postcode -- the same
|
||||||
multiplicity the tree-density filter uses for features near more than one
|
multiplicity the tree-density filter uses for features near more than one
|
||||||
postcode. The wide 100m buffer deliberately smooths police.uk's snap-to-grid
|
postcode. The 50m buffer deliberately smooths police.uk's snap-to-grid
|
||||||
coordinates, which would otherwise make the count hypersensitive to which side of
|
coordinates, which would otherwise make the count hypersensitive to which side of
|
||||||
a narrow line a shared "map point" anchor happened to land on.
|
a narrow line a shared "map point" anchor happened to land on.
|
||||||
|
|
||||||
Counts are **area-normalised**: each postcode's count is divided by its buffered
|
One figure is produced for every postcode and crime type, averaged over the
|
||||||
catchment area and rescaled by the median catchment area, so the metric reflects
|
**last 7 years** and the **last 2 years**:
|
||||||
crime *density* rather than how much ground the buffer sweeps (a median-sized
|
|
||||||
catchment is left unchanged; a large rural postcode is no longer inflated simply
|
* the **average annual incident count** -- ``sum(counts in covered window) * 12 /
|
||||||
for covering more of the map). Normalising by the buffered area -- the region
|
covered_months`` -- the raw, absolute number of recorded incidents per year,
|
||||||
that actually collects points -- rather than the raw polygon keeps tiny unit
|
with no per-area or per-capita normalisation. A covered postcode with no
|
||||||
postcodes from being over-inflated by the fixed buffer-ring floor. NOTE: this is
|
incidents of a type gets ``0``; a postcode whose force never published in the
|
||||||
an incident *density of the surrounding streets*, not a per-resident risk --
|
window, or whose geometry is unusable, gets a *null* (unknown, never a zero).
|
||||||
zero-resident commercial centres (Soho, retail parks) legitimately rank high.
|
|
||||||
|
This figure is what the server exposes as the filterable crime feature -- the
|
||||||
|
headline metric in the right pane. (An earlier version divided it by an ambient
|
||||||
|
daytime population to get a per-1,000-people rate; that was hard to read, so the
|
||||||
|
absolute per-year count is now used directly.)
|
||||||
|
|
||||||
**Force-coverage calendar.** police.uk has multi-year publication gaps for whole
|
**Force-coverage calendar.** police.uk has multi-year publication gaps for whole
|
||||||
forces (Greater Manchester has published nothing between 2019-07 and the present
|
forces (Greater Manchester has published nothing between 2019-07 and the present
|
||||||
|
|
@ -29,28 +33,25 @@ computed against the months the postcode's own force actually published:
|
||||||
matched it (BTP, which reports nationwide, is excluded from the vote);
|
matched it (BTP, which reports nationwide, is excluded from the vote);
|
||||||
postcodes with no incidents inherit their outcode's majority force, then the
|
postcodes with no incidents inherit their outcode's majority force, then the
|
||||||
national modal force.
|
national modal force.
|
||||||
* The headline ``"{type} (avg/yr)"`` is the POOLED annualised rate over the
|
* A window's average pools the counts over the force's *covered* months inside
|
||||||
force's covered months: ``sum(counts in covered years) * 12 / covered_months``.
|
that window and annualises by those months, so a coverage gap shrinks the data
|
||||||
Years in which the force published nothing contribute neither incidents nor
|
rather than reading as a low-crime dip.
|
||||||
months, so a coverage gap no longer reads as a low-crime period. (Pooling over
|
* The by-year series only emits bars for years with at least ``min_bar_months``
|
||||||
covered months also fixes the old "divide by years-with-incidents" headline,
|
covered months (default 6).
|
||||||
which inflated sporadic categories by up to ~15x.)
|
|
||||||
* The by-year series only emits bars for years with at least
|
|
||||||
``min_bar_months`` covered months (default 6): annualising a single observed
|
|
||||||
month x12 produced misleading spikes. Each bar is scaled by the force's
|
|
||||||
covered months in that year, not the global month calendar.
|
|
||||||
* ``covered_years`` (list[struct{year, months}]) is written for every postcode
|
* ``covered_years`` (list[struct{year, months}]) is written for every postcode
|
||||||
so the server can tell "covered, zero crime" (year listed, no bar) from "no
|
so the server can tell "covered, zero crime" from "no data".
|
||||||
data" (year absent) instead of charting gaps as zeros.
|
|
||||||
* Postcodes whose boundary buffer is unusable (broken geometry) get null
|
* Postcodes whose boundary buffer is unusable (broken geometry) get null
|
||||||
headline columns and an empty ``covered_years`` -- unknown, not zero.
|
figures and an empty ``covered_years`` -- unknown, not zero.
|
||||||
|
|
||||||
Outputs mirror the old LSOA transform's shape but are keyed on ``postcode``:
|
Outputs, all keyed on ``postcode``:
|
||||||
|
|
||||||
* ``crime_by_postcode.parquet`` -- ``postcode`` + ``"{type} (avg/yr)"`` columns.
|
* ``crime_by_postcode.parquet`` -- ``"{type} (/yr, 7y)"`` / ``"{type} (/yr, 2y)"``
|
||||||
* ``crime_by_postcode_by_year.parquet`` -- one row per postcode: ``postcode`` +
|
average-annual-count columns (the filterable crime features).
|
||||||
``covered_years`` + nested ``"{type} (by year)"`` ``list[struct{year, count}]``
|
* ``crime_by_postcode_by_year.parquet`` -- ``covered_years`` + nested
|
||||||
columns, with Serious/Minor rollups.
|
``"{type} (by year)"`` ``list[struct{year, count}]`` per-year raw counts.
|
||||||
|
* ``crime_records.parquet`` -- one row per counted incident over the last 7
|
||||||
|
years (``postcode`` + month/type/location/outcome/lat/lon), sorted by
|
||||||
|
postcode so the server can slice each postcode's incidents directly.
|
||||||
|
|
||||||
Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
|
Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
|
||||||
points", not true locations, and a share of rows have no coordinate at all
|
points", not true locations, and a share of rows have no coordinate at all
|
||||||
|
|
@ -63,6 +64,7 @@ from __future__ import annotations
|
||||||
import argparse
|
import argparse
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
@ -80,22 +82,36 @@ from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
|
||||||
|
|
||||||
# Serious types first so column order is stable and self-documenting.
|
# Serious types first so column order is stable and self-documenting.
|
||||||
ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
|
ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
|
||||||
|
# Output type axis = the 14 leaf types plus the two rollups, in that order.
|
||||||
|
ROLLUP_TYPES: tuple[str, ...] = ("Serious crime", "Minor crime")
|
||||||
|
ALL_OUTPUT_TYPES: tuple[str, ...] = ALL_CRIME_TYPES + ROLLUP_TYPES
|
||||||
|
|
||||||
DEFAULT_BUFFER_M = 100.0
|
DEFAULT_BUFFER_M = 50.0
|
||||||
MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
|
MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
|
||||||
STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
|
STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
|
||||||
|
|
||||||
|
# Trailing-window definitions, in (label, years) form. Each window's average is
|
||||||
|
# pooled over the force's covered months inside the window; one average-annual-
|
||||||
|
# count column (`(/yr, <label>)`) — the filterable crime feature — is emitted per
|
||||||
|
# window.
|
||||||
|
WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))
|
||||||
|
|
||||||
|
# Per-incident records cover the longest window's calendar years so the
|
||||||
|
# "individual crimes" list reconciles exactly with the headline counts: every
|
||||||
|
# record is an incident inside that window and vice versa. The headline counts
|
||||||
|
# are pooled per calendar year, so the records window must be calendar-year
|
||||||
|
# aligned too -- a rolling month span (e.g. a fixed 84 months) would include
|
||||||
|
# incidents from a year the headline excludes when the latest month is mid-year.
|
||||||
|
RECORDS_WINDOW_YEARS = max(win_years for _, win_years in WINDOWS)
|
||||||
|
|
||||||
# Minimum covered months for a year to get a by-year chart bar (and to be
|
# Minimum covered months for a year to get a by-year chart bar (and to be
|
||||||
# listed in `covered_years`). Annualising fewer observed months (x12 from a
|
# listed in `covered_years`). Annualising fewer observed months (x12 from a
|
||||||
# single month at the worst) produces bars dominated by noise, and the first
|
# single month at the worst) produces bars dominated by noise. Six months keeps
|
||||||
# (2010: one month) and current partial year would otherwise always chart as
|
# the annualisation factor <= 2.
|
||||||
# spikes/dips. Six months keeps the annualisation factor <= 2.
|
|
||||||
MIN_BAR_MONTHS = 6
|
MIN_BAR_MONTHS = 6
|
||||||
|
|
||||||
# Forces that report nationwide rather than policing a territory. They never
|
# Forces that report nationwide rather than policing a territory. They never
|
||||||
# define a postcode's home force (their publication calendar says nothing about
|
# define a postcode's home force, but their incidents still count.
|
||||||
# whether the *territorial* force covering the postcode published), but their
|
|
||||||
# incidents still count toward whichever postcodes they fall in.
|
|
||||||
NON_TERRITORIAL_FORCES = frozenset({"btp"})
|
NON_TERRITORIAL_FORCES = frozenset({"btp"})
|
||||||
|
|
||||||
COVERAGE_COLUMN = "covered_years"
|
COVERAGE_COLUMN = "covered_years"
|
||||||
|
|
@ -106,8 +122,14 @@ LON_BOUNDS = (-9.5, 2.5)
|
||||||
LAT_BOUNDS = (49.0, 61.5)
|
LAT_BOUNDS = (49.0, 61.5)
|
||||||
|
|
||||||
# Read CSVs in chunks of files to bound peak memory while keeping the STRtree
|
# Read CSVs in chunks of files to bound peak memory while keeping the STRtree
|
||||||
# query vectorised over a useful number of points.
|
# query vectorised over a useful number of points. Kept modest because the
|
||||||
_CSV_BATCH = 64
|
# in-window batches also materialise the per-incident record strings.
|
||||||
|
_CSV_BATCH = 32
|
||||||
|
|
||||||
|
|
||||||
|
def crime_column(type_name: str, window: str) -> str:
|
||||||
|
"""Filterable average-annual-count column, e.g. ``"Burglary (/yr, 7y)"``."""
|
||||||
|
return f"{type_name} (/yr, {window})"
|
||||||
|
|
||||||
|
|
||||||
def _force_calendar(
|
def _force_calendar(
|
||||||
|
|
@ -115,12 +137,9 @@ def _force_calendar(
|
||||||
) -> tuple[list[int], list[str], np.ndarray]:
|
) -> tuple[list[int], list[str], np.ndarray]:
|
||||||
"""Derive the per-force publication calendar from the CSV paths.
|
"""Derive the per-force publication calendar from the CSV paths.
|
||||||
|
|
||||||
Each police.uk file lives under ``{crime_dir}/{YYYY-MM}/{YYYY-MM}-{force}-
|
File presence IS the coverage signal: a (force, month) with no file
|
||||||
street.csv`` and holds that force's incidents for that month, so file
|
published nothing. Returns the sorted distinct years, the force slugs
|
||||||
presence IS the coverage signal: a (force, month) with no file published
|
(sorted), and ``months_in_year_force`` of shape (n_forces, n_years).
|
||||||
nothing. Returns the sorted distinct years, the force slugs (sorted), and
|
|
||||||
``months_in_year_force`` of shape (n_forces, n_years) -- how many months
|
|
||||||
each force published in each year.
|
|
||||||
"""
|
"""
|
||||||
month_force: set[tuple[str, str]] = set()
|
month_force: set[tuple[str, str]] = set()
|
||||||
for path in csvs:
|
for path in csvs:
|
||||||
|
|
@ -142,9 +161,6 @@ def _force_calendar(
|
||||||
for month, force in month_force:
|
for month, force in month_force:
|
||||||
months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1
|
months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1
|
||||||
|
|
||||||
# Surface coverage gaps loudly: any territorial force missing months inside
|
|
||||||
# the global publication window is exactly the data hole the coverage
|
|
||||||
# masking exists for.
|
|
||||||
all_months = {month for month, _ in month_force}
|
all_months = {month for month, _ in month_force}
|
||||||
for force in forces:
|
for force in forces:
|
||||||
published = {m for m, f in month_force if f == force}
|
published = {m for m, f in month_force if f == force}
|
||||||
|
|
@ -159,22 +175,25 @@ def _force_calendar(
|
||||||
|
|
||||||
def _build_tree(
|
def _build_tree(
|
||||||
polygons: np.ndarray, buffer_m: float
|
polygons: np.ndarray, buffer_m: float
|
||||||
) -> tuple[np.ndarray, shapely.STRtree]:
|
) -> tuple[shapely.STRtree, np.ndarray]:
|
||||||
"""Buffer postcode polygons outward by ``buffer_m`` and index them.
|
"""Buffer postcode polygons outward by ``buffer_m`` and index them.
|
||||||
|
|
||||||
Buffer index == postcode index. Geometries that fail to buffer are replaced
|
Buffer index == postcode index. Returns the STRtree and a ``usable`` boolean
|
||||||
with an empty polygon so the index stays aligned; they simply never match.
|
mask: geometries that fail to buffer are replaced with an empty polygon (so
|
||||||
|
the index stays aligned and they never match) and marked unusable, so their
|
||||||
|
crime picture is reported as unknown rather than zero.
|
||||||
"""
|
"""
|
||||||
buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
|
buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
|
||||||
broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
|
broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
|
||||||
if broken.any():
|
if broken.any():
|
||||||
print(f" {int(broken.sum()):,} postcode buffers unusable; left empty")
|
print(f" {int(broken.sum()):,} postcode buffers unusable; left empty")
|
||||||
buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
|
buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
|
||||||
return buffers, shapely.STRtree(buffers)
|
return shapely.STRtree(buffers), ~broken
|
||||||
|
|
||||||
|
|
||||||
def _accumulate_counts(
|
def _accumulate_counts(
|
||||||
csvs: list[Path],
|
csvs: list[Path],
|
||||||
|
postcodes: np.ndarray,
|
||||||
tree: shapely.STRtree,
|
tree: shapely.STRtree,
|
||||||
type_to_idx: dict[str, int],
|
type_to_idx: dict[str, int],
|
||||||
year_to_idx: dict[int, int],
|
year_to_idx: dict[int, int],
|
||||||
|
|
@ -182,35 +201,49 @@ def _accumulate_counts(
|
||||||
transformer: Transformer,
|
transformer: Transformer,
|
||||||
counts: np.ndarray,
|
counts: np.ndarray,
|
||||||
force_votes: np.ndarray,
|
force_votes: np.ndarray,
|
||||||
|
records_shard_dir: Path | None,
|
||||||
|
records_min_ym: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Stream the crime CSVs, counting points-in-buffer per (postcode, type, year).
|
"""Stream the crime CSVs, counting points-in-buffer per (postcode, type, year).
|
||||||
|
|
||||||
Also accumulates ``force_votes`` (n_postcodes, n_forces): how many matched
|
Also accumulates ``force_votes`` (n_postcodes, n_forces) for the home-force
|
||||||
incidents each force's files contributed to each postcode, which later
|
election and, when ``records_shard_dir`` is set, writes one parquet shard per
|
||||||
elects the postcode's home force for the coverage calendar.
|
batch holding every counted (incident, postcode) pair whose month is within
|
||||||
|
the records window (month index >= ``records_min_ym``).
|
||||||
"""
|
"""
|
||||||
|
# Type overrides only for the columns we ever read; LSOA is not stored.
|
||||||
schema = {
|
schema = {
|
||||||
"Longitude": pl.Float64,
|
"Longitude": pl.Float64,
|
||||||
"Latitude": pl.Float64,
|
"Latitude": pl.Float64,
|
||||||
"Month": pl.Utf8,
|
"Month": pl.Utf8,
|
||||||
"Crime type": pl.Utf8,
|
"Crime type": pl.Utf8,
|
||||||
|
"Location": pl.Utf8,
|
||||||
|
"Last outcome category": pl.Utf8,
|
||||||
}
|
}
|
||||||
years = list(year_to_idx)
|
years = list(year_to_idx)
|
||||||
total_points = 0
|
total_points = 0
|
||||||
total_matches = 0
|
total_matches = 0
|
||||||
total_dropped = 0
|
total_dropped = 0
|
||||||
|
total_records = 0
|
||||||
unknown_type_counts: dict[str, int] = {}
|
unknown_type_counts: dict[str, int] = {}
|
||||||
|
|
||||||
for start in range(0, len(csvs), _CSV_BATCH):
|
for start in range(0, len(csvs), _CSV_BATCH):
|
||||||
batch = csvs[start : start + _CSV_BATCH]
|
batch = csvs[start : start + _CSV_BATCH]
|
||||||
# The source file identifies the publishing force (police.uk has no
|
|
||||||
# force column with consistent naming); map each path back to its
|
|
||||||
# force index for the home-force vote.
|
|
||||||
path_to_fidx = {}
|
path_to_fidx = {}
|
||||||
|
batch_max_ym = -1
|
||||||
for path in batch:
|
for path in batch:
|
||||||
m = STREET_CSV_NAME_RE.fullmatch(path.name)
|
m = STREET_CSV_NAME_RE.fullmatch(path.name)
|
||||||
if m is not None and m.group(2) in force_to_idx:
|
if m is not None:
|
||||||
path_to_fidx[str(path)] = force_to_idx[m.group(2)]
|
ym = m.group(1)
|
||||||
|
batch_max_ym = max(batch_max_ym, int(ym[:4]) * 12 + int(ym[5:7]) - 1)
|
||||||
|
if m.group(2) in force_to_idx:
|
||||||
|
path_to_fidx[str(path)] = force_to_idx[m.group(2)]
|
||||||
|
# The per-incident record strings (Location, outcome) are by far the
|
||||||
|
# heaviest columns; read them only for batches that fall inside the
|
||||||
|
# records window, so the ~50% of pre-window months cost nothing extra.
|
||||||
|
want_records = records_shard_dir is not None and batch_max_ym >= records_min_ym
|
||||||
|
record_cols = ["Location", "Last outcome category"] if want_records else []
|
||||||
|
|
||||||
frame = (
|
frame = (
|
||||||
pl.scan_csv(
|
pl.scan_csv(
|
||||||
batch,
|
batch,
|
||||||
|
|
@ -218,12 +251,12 @@ def _accumulate_counts(
|
||||||
ignore_errors=True,
|
ignore_errors=True,
|
||||||
include_file_paths="_source_path",
|
include_file_paths="_source_path",
|
||||||
)
|
)
|
||||||
.select("Longitude", "Latitude", "Month", "Crime type", "_source_path")
|
.select(
|
||||||
# strict=False: a single malformed Month drops only that row instead
|
"Longitude", "Latitude", "Month", "Crime type", *record_cols, "_source_path"
|
||||||
# of aborting the whole build (a non-numeric year becomes null and is
|
)
|
||||||
# filtered out by the year membership check below).
|
|
||||||
.with_columns(
|
.with_columns(
|
||||||
pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year")
|
pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year"),
|
||||||
|
pl.col("Month").str.slice(5, 2).cast(pl.Int32, strict=False).alias("_mm"),
|
||||||
)
|
)
|
||||||
.filter(
|
.filter(
|
||||||
pl.col("Longitude").is_not_null()
|
pl.col("Longitude").is_not_null()
|
||||||
|
|
@ -234,14 +267,11 @@ def _accumulate_counts(
|
||||||
& (pl.col("Crime type") != "")
|
& (pl.col("Crime type") != "")
|
||||||
& pl.col("year").is_in(years)
|
& pl.col("year").is_in(years)
|
||||||
)
|
)
|
||||||
# Canonicalise legacy pre-2014 crime-type names ("Violent crime",
|
# year*12 + (month-1): an integer month index for window filtering.
|
||||||
# "Public disorder and weapons") to their current equivalents before
|
.with_columns(
|
||||||
# indexing, so ~1.9M historical incidents are counted instead of
|
(pl.col("year") * 12 + (pl.col("_mm").fill_null(1) - 1)).alias("month_index")
|
||||||
# dropped. `.replace` leaves current types unchanged.
|
)
|
||||||
.with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
|
.with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
|
||||||
# Map crime types to indices with default=None so an unrecognised
|
|
||||||
# type yields a null index we can *report* rather than silently drop
|
|
||||||
# (the legacy LSOA path surfaced unknown types via its dynamic pivot).
|
|
||||||
.with_columns(
|
.with_columns(
|
||||||
pl.col("Crime type")
|
pl.col("Crime type")
|
||||||
.replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
|
.replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
|
||||||
|
|
@ -253,7 +283,16 @@ def _accumulate_counts(
|
||||||
.replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32)
|
.replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32)
|
||||||
.alias("fidx"),
|
.alias("fidx"),
|
||||||
)
|
)
|
||||||
.select("Longitude", "Latitude", "Crime type", "tidx", "yidx", "fidx")
|
.select(
|
||||||
|
"Longitude",
|
||||||
|
"Latitude",
|
||||||
|
"Crime type",
|
||||||
|
*record_cols,
|
||||||
|
"month_index",
|
||||||
|
"tidx",
|
||||||
|
"yidx",
|
||||||
|
"fidx",
|
||||||
|
)
|
||||||
.collect(engine="streaming")
|
.collect(engine="streaming")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -273,19 +312,15 @@ def _accumulate_counts(
|
||||||
tidx = frame["tidx"].to_numpy()
|
tidx = frame["tidx"].to_numpy()
|
||||||
yidx = frame["yidx"].to_numpy()
|
yidx = frame["yidx"].to_numpy()
|
||||||
fidx = frame["fidx"].to_numpy()
|
fidx = frame["fidx"].to_numpy()
|
||||||
|
month_index = frame["month_index"].to_numpy()
|
||||||
|
|
||||||
x, y = transformer.transform(lon, lat)
|
x, y = transformer.transform(lon, lat)
|
||||||
finite = np.isfinite(x) & np.isfinite(y)
|
finite = np.isfinite(x) & np.isfinite(y)
|
||||||
total_dropped += int((~finite).sum())
|
total_dropped += int((~finite).sum())
|
||||||
if not finite.any():
|
if not finite.any():
|
||||||
continue
|
continue
|
||||||
x, y, tidx, yidx, fidx = (
|
x, y = x[finite], y[finite]
|
||||||
x[finite],
|
tidx, yidx, fidx = tidx[finite], yidx[finite], fidx[finite]
|
||||||
y[finite],
|
|
||||||
tidx[finite],
|
|
||||||
yidx[finite],
|
|
||||||
fidx[finite],
|
|
||||||
)
|
|
||||||
total_points += x.size
|
total_points += x.size
|
||||||
|
|
||||||
points = shapely.points(x, y)
|
points = shapely.points(x, y)
|
||||||
|
|
@ -306,9 +341,26 @@ def _accumulate_counts(
|
||||||
)
|
)
|
||||||
total_matches += point_index.size
|
total_matches += point_index.size
|
||||||
|
|
||||||
|
if want_records:
|
||||||
|
total_records += _write_record_shard(
|
||||||
|
records_shard_dir,
|
||||||
|
start,
|
||||||
|
postcodes,
|
||||||
|
point_index,
|
||||||
|
postcode_index,
|
||||||
|
month_index[finite],
|
||||||
|
frame["Crime type"].to_numpy()[finite],
|
||||||
|
frame["Location"].to_numpy()[finite],
|
||||||
|
frame["Last outcome category"].to_numpy()[finite],
|
||||||
|
lon[finite],
|
||||||
|
lat[finite],
|
||||||
|
records_min_ym,
|
||||||
|
)
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f" files {start + len(batch):,}/{len(csvs):,}: "
|
f" files {start + len(batch):,}/{len(csvs):,}: "
|
||||||
f"{total_points:,} located points, {total_matches:,} postcode matches"
|
f"{total_points:,} located points, {total_matches:,} postcode matches"
|
||||||
|
+ (f", {total_records:,} records" if records_shard_dir is not None else "")
|
||||||
)
|
)
|
||||||
|
|
||||||
if total_dropped:
|
if total_dropped:
|
||||||
|
|
@ -329,19 +381,65 @@ def _accumulate_counts(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_record_shard(
|
||||||
|
records_shard_dir: Path,
|
||||||
|
start: int,
|
||||||
|
postcodes: np.ndarray,
|
||||||
|
point_index: np.ndarray,
|
||||||
|
postcode_index: np.ndarray,
|
||||||
|
month_index: np.ndarray,
|
||||||
|
crime_type: np.ndarray,
|
||||||
|
location: np.ndarray,
|
||||||
|
outcome: np.ndarray,
|
||||||
|
lon: np.ndarray,
|
||||||
|
lat: np.ndarray,
|
||||||
|
records_min_ym: int,
|
||||||
|
) -> int:
|
||||||
|
"""Write one parquet shard of (incident, postcode) records for this batch.
|
||||||
|
|
||||||
|
Each matched pair becomes a row -- the same multiplicity as the count -- so a
|
||||||
|
postcode's records are exactly the incidents that made up its counts. Only
|
||||||
|
incidents within the records window (month index >= ``records_min_ym``) are
|
||||||
|
kept. Returns the number of rows written.
|
||||||
|
"""
|
||||||
|
mi = month_index[point_index]
|
||||||
|
keep = mi >= records_min_ym
|
||||||
|
if not keep.any():
|
||||||
|
return 0
|
||||||
|
pidx = point_index[keep]
|
||||||
|
# Build the string columns from Python lists (.tolist()) rather than numpy
|
||||||
|
# object arrays: an all-null Location/outcome slice is an all-None object
|
||||||
|
# array that polars cannot cast to String, whereas a list of str|None infers
|
||||||
|
# a nullable String column cleanly.
|
||||||
|
shard = pl.DataFrame(
|
||||||
|
{
|
||||||
|
"postcode": postcodes[postcode_index[keep]].astype(str),
|
||||||
|
"month_index": mi[keep].astype(np.int32),
|
||||||
|
"crime_type": crime_type[pidx].astype(str),
|
||||||
|
"location": location[pidx].tolist(),
|
||||||
|
"outcome": outcome[pidx].tolist(),
|
||||||
|
"lat": lat[pidx].astype(np.float32),
|
||||||
|
"lon": lon[pidx].astype(np.float32),
|
||||||
|
},
|
||||||
|
schema_overrides={
|
||||||
|
"postcode": pl.String,
|
||||||
|
"crime_type": pl.String,
|
||||||
|
"location": pl.String,
|
||||||
|
"outcome": pl.String,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
shard.write_parquet(
|
||||||
|
records_shard_dir / f"{start:08d}.parquet", compression="zstd"
|
||||||
|
)
|
||||||
|
return shard.height
|
||||||
|
|
||||||
|
|
||||||
def _assign_home_force(
|
def _assign_home_force(
|
||||||
postcodes: np.ndarray,
|
postcodes: np.ndarray,
|
||||||
force_votes: np.ndarray,
|
force_votes: np.ndarray,
|
||||||
forces: list[str],
|
forces: list[str],
|
||||||
) -> np.ndarray:
|
) -> np.ndarray:
|
||||||
"""Elect each postcode's home (territorial) force.
|
"""Elect each postcode's home (territorial) force by majority incident vote."""
|
||||||
|
|
||||||
Majority vote of matched incidents per publishing force; non-territorial
|
|
||||||
forces (BTP) are excluded from the vote because their calendar says nothing
|
|
||||||
about local coverage. Postcodes with no votes (no incidents ever, or
|
|
||||||
BTP-only) inherit the majority force of their outcode, then the national
|
|
||||||
modal force, so every postcode gets a coverage calendar.
|
|
||||||
"""
|
|
||||||
votes = force_votes.astype(np.int64, copy=True)
|
votes = force_votes.astype(np.int64, copy=True)
|
||||||
for idx, force in enumerate(forces):
|
for idx, force in enumerate(forces):
|
||||||
if force in NON_TERRITORIAL_FORCES:
|
if force in NON_TERRITORIAL_FORCES:
|
||||||
|
|
@ -354,18 +452,15 @@ def _assign_home_force(
|
||||||
if not has_vote.any():
|
if not has_vote.any():
|
||||||
raise ValueError("No incidents matched any postcode; cannot assign forces")
|
raise ValueError("No incidents matched any postcode; cannot assign forces")
|
||||||
|
|
||||||
# Outcode-majority fallback for postcodes with no (territorial) incidents.
|
|
||||||
outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object)
|
outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object)
|
||||||
national_modal = int(
|
national_modal = int(np.bincount(home[has_vote], minlength=len(forces)).argmax())
|
||||||
np.bincount(home[has_vote], minlength=len(forces)).argmax()
|
|
||||||
)
|
|
||||||
if (~has_vote).any():
|
if (~has_vote).any():
|
||||||
outcode_modal: dict[str, int] = {}
|
outcode_modal: dict[str, int] = {}
|
||||||
voted_outcodes = outcodes[has_vote]
|
voted_outcodes = outcodes[has_vote]
|
||||||
voted_home = home[has_vote]
|
voted_home = home[has_vote]
|
||||||
for oc in np.unique(voted_outcodes):
|
for oc in np.unique(voted_outcodes):
|
||||||
counts = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
|
tally = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
|
||||||
outcode_modal[oc] = int(counts.argmax())
|
outcode_modal[oc] = int(tally.argmax())
|
||||||
fallback = np.array(
|
fallback = np.array(
|
||||||
[outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]],
|
[outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]],
|
||||||
dtype=np.int32,
|
dtype=np.int32,
|
||||||
|
|
@ -379,10 +474,82 @@ def _assign_home_force(
|
||||||
return home
|
return home
|
||||||
|
|
||||||
|
|
||||||
|
def _window_annualised(
|
||||||
|
counts: np.ndarray,
|
||||||
|
months_in_year_force: np.ndarray,
|
||||||
|
home_fidx: np.ndarray,
|
||||||
|
usable: np.ndarray,
|
||||||
|
year_mask: np.ndarray,
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""Raw annualised incidents/yr per (postcode, type) over a window of years.
|
||||||
|
|
||||||
|
For each force, the count is pooled over the force's covered months that fall
|
||||||
|
inside ``year_mask`` and annualised by those covered months. A covered
|
||||||
|
postcode with no incidents of a type gets 0; a postcode whose force never
|
||||||
|
published in the window, or whose geometry is unusable, gets NaN (unknown).
|
||||||
|
"""
|
||||||
|
n_pc, n_types = counts.shape[0], counts.shape[1]
|
||||||
|
avg = np.full((n_pc, n_types), np.nan, dtype=np.float64)
|
||||||
|
for f in range(months_in_year_force.shape[0]):
|
||||||
|
sel = home_fidx == f
|
||||||
|
if not sel.any():
|
||||||
|
continue
|
||||||
|
cov_months = months_in_year_force[f].astype(np.float64) * year_mask
|
||||||
|
denom = cov_months.sum()
|
||||||
|
if denom <= 0:
|
||||||
|
continue # force published nothing in this window; stays NaN
|
||||||
|
window_years = cov_months > 0
|
||||||
|
pooled = counts[sel][:, :, window_years].sum(axis=2, dtype=np.float64)
|
||||||
|
avg[sel] = pooled * 12.0 / denom
|
||||||
|
avg[~usable] = np.nan
|
||||||
|
return avg
|
||||||
|
|
||||||
|
|
||||||
|
def _append_rollups(avg14: np.ndarray) -> np.ndarray:
|
||||||
|
"""Append Serious/Minor rollup columns (sum of components) -> (n_pc, 16)."""
|
||||||
|
serious_idx = [ALL_CRIME_TYPES.index(t) for t in SERIOUS_CRIME_TYPES]
|
||||||
|
minor_idx = [ALL_CRIME_TYPES.index(t) for t in MINOR_CRIME_TYPES]
|
||||||
|
serious = avg14[:, serious_idx].sum(axis=1)
|
||||||
|
minor = avg14[:, minor_idx].sum(axis=1)
|
||||||
|
return np.column_stack([avg14, serious, minor])
|
||||||
|
|
||||||
|
|
||||||
|
def _write_crime_table(
|
||||||
|
postcodes: np.ndarray,
|
||||||
|
raw_by_window: dict[str, np.ndarray],
|
||||||
|
output_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""Write the average-annual-count parquet (the filterable crime features).
|
||||||
|
|
||||||
|
``raw_by_window[label]`` is the (n_postcodes, 16) average annual incident
|
||||||
|
count for that window (14 leaf types + 2 rollups). Each value is the raw,
|
||||||
|
absolute incidents/yr; a postcode with no usable data for a window keeps NaN
|
||||||
|
(written as null).
|
||||||
|
"""
|
||||||
|
data: dict[str, np.ndarray] = {"postcode": postcodes}
|
||||||
|
for label, _years in WINDOWS:
|
||||||
|
counts = np.round(raw_by_window[label], 1).astype(np.float32)
|
||||||
|
for type_idx, name in enumerate(ALL_OUTPUT_TYPES):
|
||||||
|
data[crime_column(name, label)] = counts[:, type_idx]
|
||||||
|
|
||||||
|
_write_nan_aware(data, output_path, "postcode crime average annual counts")
|
||||||
|
|
||||||
|
|
||||||
|
def _write_nan_aware(
|
||||||
|
data: dict[str, np.ndarray], output_path: Path, label: str
|
||||||
|
) -> None:
|
||||||
|
frame = pl.DataFrame(data)
|
||||||
|
value_cols = [c for c in frame.columns if c != "postcode"]
|
||||||
|
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
frame.write_parquet(output_path, compression="zstd")
|
||||||
|
print(f"Wrote {label}: {output_path} {frame.shape}")
|
||||||
|
|
||||||
|
|
||||||
def _rollup_long(
|
def _rollup_long(
|
||||||
long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
|
long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
|
||||||
) -> pl.DataFrame:
|
) -> pl.DataFrame:
|
||||||
"""Sum per-year annualised counts across ``types`` into a single rollup."""
|
"""Sum per-year counts across ``types`` into a single rollup."""
|
||||||
return (
|
return (
|
||||||
long.filter(pl.col("Crime type").is_in(list(types)))
|
long.filter(pl.col("Crime type").is_in(list(types)))
|
||||||
.group_by("postcode", "year")
|
.group_by("postcode", "year")
|
||||||
|
|
@ -392,108 +559,29 @@ def _rollup_long(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _write_avg_yr(
|
|
||||||
postcodes: np.ndarray,
|
|
||||||
counts: np.ndarray,
|
|
||||||
months_in_year_force: np.ndarray,
|
|
||||||
home_fidx: np.ndarray,
|
|
||||||
norm: np.ndarray,
|
|
||||||
output_path: Path,
|
|
||||||
) -> None:
|
|
||||||
"""Write ``postcode`` + ``"{type} (avg/yr)"`` density-normalised averages.
|
|
||||||
|
|
||||||
The headline is the POOLED annualised rate over the home force's covered
|
|
||||||
months: ``sum(counts in covered years) * 12 / covered_months``. Years the
|
|
||||||
force published nothing contribute neither incidents nor months, so a
|
|
||||||
coverage gap (e.g. Greater Manchester 2019-07 onwards) is excluded instead
|
|
||||||
of read as zero crime. Pooling over the full covered window -- rather than
|
|
||||||
averaging only over years a type happened to occur -- is what keeps a
|
|
||||||
single robbery-year from printing as a perennial robbery rate. Each
|
|
||||||
postcode's value is then multiplied by ``norm`` (median_area / buffered
|
|
||||||
catchment area) so the metric is a density rather than a footprint-inflated
|
|
||||||
raw count; postcodes with unusable geometry (norm == 0) are null, not 0.
|
|
||||||
"""
|
|
||||||
n_postcodes, n_types = counts.shape[0], counts.shape[1]
|
|
||||||
avg = np.full((n_postcodes, n_types), np.nan, dtype=np.float64)
|
|
||||||
for f in range(months_in_year_force.shape[0]):
|
|
||||||
sel = home_fidx == f
|
|
||||||
if not sel.any():
|
|
||||||
continue
|
|
||||||
cov_months = months_in_year_force[f].astype(np.float64)
|
|
||||||
denom = cov_months.sum()
|
|
||||||
if denom <= 0:
|
|
||||||
continue # force never published; stays null
|
|
||||||
covered_years = cov_months > 0
|
|
||||||
pooled = counts[sel][:, :, covered_years].sum(axis=2, dtype=np.float64)
|
|
||||||
avg[sel] = pooled * 12.0 / denom
|
|
||||||
|
|
||||||
avg *= norm[:, None]
|
|
||||||
avg[norm <= 0] = np.nan # unusable geometry: unknown, not zero
|
|
||||||
avg = np.round(avg, 1).astype(np.float32)
|
|
||||||
|
|
||||||
data: dict[str, np.ndarray] = {"postcode": postcodes}
|
|
||||||
for type_idx, name in enumerate(ALL_CRIME_TYPES):
|
|
||||||
data[f"{name} (avg/yr)"] = avg[:, type_idx]
|
|
||||||
|
|
||||||
# Serious/Minor rollup headlines = the exact SUM of their component (avg/yr)
|
|
||||||
# columns, so each rollup always equals the sum of the parts shown beside it
|
|
||||||
# and can never fall below one of its own components. All components share
|
|
||||||
# the postcode's pooled covered-month denominator, so the sum is itself the
|
|
||||||
# pooled rollup rate. Null components (unusable geometry) propagate to a
|
|
||||||
# null rollup.
|
|
||||||
for rollup_name, rollup_types in (
|
|
||||||
("Serious crime", SERIOUS_CRIME_TYPES),
|
|
||||||
("Minor crime", MINOR_CRIME_TYPES),
|
|
||||||
):
|
|
||||||
rollup_idx = [ALL_CRIME_TYPES.index(name) for name in rollup_types]
|
|
||||||
data[f"{rollup_name} (avg/yr)"] = np.round(
|
|
||||||
avg[:, rollup_idx].sum(axis=1), 1
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
frame = pl.DataFrame(data)
|
|
||||||
value_cols = [c for c in frame.columns if c != "postcode"]
|
|
||||||
frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
|
|
||||||
|
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
frame.write_parquet(output_path, compression="zstd")
|
|
||||||
print(f"Wrote postcode crime averages: {output_path}")
|
|
||||||
|
|
||||||
|
|
||||||
def _write_by_year(
|
def _write_by_year(
|
||||||
postcodes: np.ndarray,
|
postcodes: np.ndarray,
|
||||||
counts: np.ndarray,
|
counts: np.ndarray,
|
||||||
years: list[int],
|
years: list[int],
|
||||||
months_in_year_force: np.ndarray,
|
months_in_year_force: np.ndarray,
|
||||||
home_fidx: np.ndarray,
|
home_fidx: np.ndarray,
|
||||||
norm: np.ndarray,
|
usable: np.ndarray,
|
||||||
min_bar_months: int,
|
min_bar_months: int,
|
||||||
output_path: Path,
|
output_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Write nested ``"{type} (by year)"`` series plus rollups and coverage.
|
"""Write nested ``"{type} (by year)"`` raw-count series plus rollups + coverage.
|
||||||
|
|
||||||
A bar is only emitted for (postcode, year)s where the postcode's home force
|
A bar is only emitted for (postcode, year)s where the postcode's home force
|
||||||
published at least ``min_bar_months`` months -- annualising a thinner year
|
published at least ``min_bar_months`` months. Bars are the raw annualised
|
||||||
(x12 from a single month at the extreme) charts noise, and a force-gap year
|
count for that year (``count * 12 / covered_months``); unlike the headline
|
||||||
must chart as *no data*, not zero. Bars are scaled by the force's covered
|
windows there is no per-capita normalisation here -- the chart shows incident
|
||||||
months in that year and area-normalised by the same ``norm`` factor as the
|
volume over time. Every postcode gets a ``covered_years`` row so consumers can
|
||||||
headline so chart and headline stay mutually consistent.
|
distinguish covered-but-crime-free years from coverage gaps.
|
||||||
|
|
||||||
Every postcode gets a row (the output is dense) carrying ``covered_years``
|
|
||||||
-- the list of {year, months} the home force published at least
|
|
||||||
``min_bar_months`` months -- so consumers can distinguish covered-but-
|
|
||||||
crime-free years (year listed, no bar => genuine zero) from coverage gaps
|
|
||||||
(year absent => unknown). Postcodes with unusable geometry get an empty
|
|
||||||
coverage list: their crime picture is unknown.
|
|
||||||
"""
|
"""
|
||||||
# (n_postcodes, n_years): covered months of each postcode's home force.
|
|
||||||
cov_pc_year = months_in_year_force[home_fidx, :]
|
cov_pc_year = months_in_year_force[home_fidx, :]
|
||||||
usable = norm > 0
|
|
||||||
|
|
||||||
annual = np.round(
|
annual = np.round(
|
||||||
counts.astype(np.float64)
|
counts.astype(np.float64) * 12.0 / np.maximum(cov_pc_year[:, None, :], 1),
|
||||||
* 12.0
|
|
||||||
/ np.maximum(cov_pc_year[:, None, :], 1)
|
|
||||||
* norm[:, None, None],
|
|
||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
bar_ok = (
|
bar_ok = (
|
||||||
|
|
@ -506,9 +594,6 @@ def _write_by_year(
|
||||||
|
|
||||||
type_names = np.array(ALL_CRIME_TYPES, dtype=object)
|
type_names = np.array(ALL_CRIME_TYPES, dtype=object)
|
||||||
year_values = np.array(years, dtype=np.int32)
|
year_values = np.array(years, dtype=np.int32)
|
||||||
# Explicit schema: with full masking (e.g. every year below min_bar_months)
|
|
||||||
# the fancy-indexed numpy object arrays are empty and polars would infer
|
|
||||||
# Object columns, which breaks the rollup `is_in` below.
|
|
||||||
long = pl.DataFrame(
|
long = pl.DataFrame(
|
||||||
{
|
{
|
||||||
"postcode": postcodes[pc_i].astype(str),
|
"postcode": postcodes[pc_i].astype(str),
|
||||||
|
|
@ -532,8 +617,6 @@ def _write_by_year(
|
||||||
type_cols = [c for c in wide.columns if c != "postcode"]
|
type_cols = [c for c in wide.columns if c != "postcode"]
|
||||||
wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
|
wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
|
||||||
|
|
||||||
# Dense base: every postcode, with its home force's coverage calendar.
|
|
||||||
# Built per force (there are ~45) and joined on the force index.
|
|
||||||
coverage_per_force: list[list[dict[str, int]]] = []
|
coverage_per_force: list[list[dict[str, int]]] = []
|
||||||
for f in range(months_in_year_force.shape[0]):
|
for f in range(months_in_year_force.shape[0]):
|
||||||
coverage_per_force.append(
|
coverage_per_force.append(
|
||||||
|
|
@ -562,7 +645,6 @@ def _write_by_year(
|
||||||
dense = (
|
dense = (
|
||||||
base.join(coverage_frame, on="_fidx", how="left")
|
base.join(coverage_frame, on="_fidx", how="left")
|
||||||
.with_columns(
|
.with_columns(
|
||||||
# Unusable geometry: empty coverage -- the crime picture is unknown.
|
|
||||||
pl.when(pl.col("_usable"))
|
pl.when(pl.col("_usable"))
|
||||||
.then(pl.col(COVERAGE_COLUMN))
|
.then(pl.col(COVERAGE_COLUMN))
|
||||||
.otherwise(pl.col(COVERAGE_COLUMN).list.head(0))
|
.otherwise(pl.col(COVERAGE_COLUMN).list.head(0))
|
||||||
|
|
@ -577,11 +659,47 @@ def _write_by_year(
|
||||||
print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}")
|
print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}")
|
||||||
|
|
||||||
|
|
||||||
|
def _finalize_records(
|
||||||
|
records_shard_dir: Path, output_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Concatenate the per-batch record shards into one postcode-sorted parquet.
|
||||||
|
|
||||||
|
Sorting by postcode lets the server build a contiguous per-postcode slice
|
||||||
|
index. The sort runs on the streaming engine so it spills rather than holding
|
||||||
|
all ~40M rows in memory.
|
||||||
|
"""
|
||||||
|
shards = sorted(records_shard_dir.glob("*.parquet"))
|
||||||
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if not shards:
|
||||||
|
pl.DataFrame(
|
||||||
|
schema={
|
||||||
|
"postcode": pl.String,
|
||||||
|
"month_index": pl.Int32,
|
||||||
|
"crime_type": pl.String,
|
||||||
|
"location": pl.String,
|
||||||
|
"outcome": pl.String,
|
||||||
|
"lat": pl.Float32,
|
||||||
|
"lon": pl.Float32,
|
||||||
|
}
|
||||||
|
).write_parquet(output_path, compression="zstd")
|
||||||
|
print(f"Wrote crime records: {output_path} (empty)")
|
||||||
|
return
|
||||||
|
|
||||||
|
(
|
||||||
|
pl.scan_parquet(shards)
|
||||||
|
.sort("postcode")
|
||||||
|
.sink_parquet(output_path, compression="zstd")
|
||||||
|
)
|
||||||
|
n = pl.scan_parquet(output_path).select(pl.len()).collect().item()
|
||||||
|
print(f"Wrote crime records: {output_path} ({n:,} rows)")
|
||||||
|
|
||||||
|
|
||||||
def transform_crime_spatial(
|
def transform_crime_spatial(
|
||||||
crime_dir: Path,
|
crime_dir: Path,
|
||||||
boundaries_dir: Path,
|
boundaries_dir: Path,
|
||||||
output_path: Path,
|
output_path: Path,
|
||||||
by_year_output_path: Path,
|
by_year_output_path: Path,
|
||||||
|
records_output_path: Path,
|
||||||
buffer_m: float = DEFAULT_BUFFER_M,
|
buffer_m: float = DEFAULT_BUFFER_M,
|
||||||
max_postcodes: int | None = None,
|
max_postcodes: int | None = None,
|
||||||
max_files: int | None = None,
|
max_files: int | None = None,
|
||||||
|
|
@ -594,6 +712,10 @@ def transform_crime_spatial(
|
||||||
csvs = csvs[:max_files]
|
csvs = csvs[:max_files]
|
||||||
|
|
||||||
years, forces, months_in_year_force = _force_calendar(csvs)
|
years, forces, months_in_year_force = _force_calendar(csvs)
|
||||||
|
latest_year = years[-1]
|
||||||
|
# Records cover the longest window's calendar years (January of its earliest
|
||||||
|
# year onward), so they reconcile exactly with the calendar-year headline.
|
||||||
|
records_min_ym = (latest_year - (RECORDS_WINDOW_YEARS - 1)) * 12
|
||||||
print(
|
print(
|
||||||
f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces "
|
f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces "
|
||||||
f"({years[0]}-{years[-1]})"
|
f"({years[0]}-{years[-1]})"
|
||||||
|
|
@ -601,27 +723,10 @@ def transform_crime_spatial(
|
||||||
)
|
)
|
||||||
|
|
||||||
postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
|
postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
|
||||||
|
postcodes = np.asarray(postcodes)
|
||||||
|
|
||||||
print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
|
print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
|
||||||
buffers, tree = _build_tree(polygons, buffer_m)
|
tree, usable = _build_tree(polygons, buffer_m)
|
||||||
|
|
||||||
# Area-normalisation factor (median_area / catchment_area): divides out the
|
|
||||||
# size of each postcode's catchment so the count measures crime density, not
|
|
||||||
# how much ground the buffer sweeps. We normalise by the *buffered* area --
|
|
||||||
# the region that actually collects points -- rather than the raw polygon, so
|
|
||||||
# a tiny unit postcode isn't over-inflated by the fixed buffer-ring floor.
|
|
||||||
# Buffers are in EPSG:27700, so shapely.area is in m^2.
|
|
||||||
areas = shapely.area(buffers).astype(np.float64)
|
|
||||||
usable_area = np.isfinite(areas) & (areas > 0)
|
|
||||||
if not usable_area.any():
|
|
||||||
raise ValueError("No postcode buffers have a positive area to normalise by")
|
|
||||||
median_area = float(np.median(areas[usable_area]))
|
|
||||||
norm = np.zeros(len(postcodes), dtype=np.float64)
|
|
||||||
norm[usable_area] = median_area / areas[usable_area]
|
|
||||||
print(
|
|
||||||
f"Area-normalising to median catchment area {median_area:,.0f} m^2 "
|
|
||||||
f"({int(usable_area.sum()):,}/{len(areas):,} postcodes have usable area)"
|
|
||||||
)
|
|
||||||
|
|
||||||
type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
|
type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
|
||||||
year_to_idx = {year: idx for idx, year in enumerate(years)}
|
year_to_idx = {year: idx for idx, year in enumerate(years)}
|
||||||
|
|
@ -630,25 +735,50 @@ def transform_crime_spatial(
|
||||||
force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32)
|
force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32)
|
||||||
|
|
||||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
|
transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
|
||||||
_accumulate_counts(
|
|
||||||
csvs, tree, type_to_idx, year_to_idx, force_to_idx, transformer, counts, force_votes
|
|
||||||
)
|
|
||||||
|
|
||||||
home_fidx = _assign_home_force(np.asarray(postcodes), force_votes, forces)
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="crime_records_", dir=records_output_path.parent
|
||||||
|
) as shard_dir_str:
|
||||||
|
shard_dir = Path(shard_dir_str)
|
||||||
|
_accumulate_counts(
|
||||||
|
csvs,
|
||||||
|
postcodes,
|
||||||
|
tree,
|
||||||
|
type_to_idx,
|
||||||
|
year_to_idx,
|
||||||
|
force_to_idx,
|
||||||
|
transformer,
|
||||||
|
counts,
|
||||||
|
force_votes,
|
||||||
|
shard_dir,
|
||||||
|
records_min_ym,
|
||||||
|
)
|
||||||
|
|
||||||
_write_avg_yr(
|
home_fidx = _assign_home_force(postcodes, force_votes, forces)
|
||||||
postcodes, counts, months_in_year_force, home_fidx, norm, output_path
|
|
||||||
)
|
# Per-window raw annualised averages (14 leaf types + Serious/Minor).
|
||||||
_write_by_year(
|
raw_by_window: dict[str, np.ndarray] = {}
|
||||||
postcodes,
|
for label, win_years in WINDOWS:
|
||||||
counts,
|
year_mask = np.array(
|
||||||
years,
|
[1.0 if y > latest_year - win_years else 0.0 for y in years]
|
||||||
months_in_year_force,
|
)
|
||||||
home_fidx,
|
avg14 = _window_annualised(
|
||||||
norm,
|
counts, months_in_year_force, home_fidx, usable, year_mask
|
||||||
min_bar_months,
|
)
|
||||||
by_year_output_path,
|
raw_by_window[label] = _append_rollups(avg14)
|
||||||
)
|
|
||||||
|
_write_crime_table(postcodes, raw_by_window, output_path)
|
||||||
|
_write_by_year(
|
||||||
|
postcodes,
|
||||||
|
counts,
|
||||||
|
years,
|
||||||
|
months_in_year_force,
|
||||||
|
home_fidx,
|
||||||
|
usable,
|
||||||
|
min_bar_months,
|
||||||
|
by_year_output_path,
|
||||||
|
)
|
||||||
|
_finalize_records(shard_dir, records_output_path)
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
|
|
@ -671,7 +801,7 @@ def main() -> None:
|
||||||
"--output",
|
"--output",
|
||||||
type=Path,
|
type=Path,
|
||||||
required=True,
|
required=True,
|
||||||
help="Output parquet: postcode + '{type} (avg/yr)' columns",
|
help="Output parquet: postcode + '{type} (/yr, <window>)' average-annual-count columns",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--output-by-year",
|
"--output-by-year",
|
||||||
|
|
@ -680,10 +810,10 @@ def main() -> None:
|
||||||
help="Output parquet: postcode + nested '{type} (by year)' columns",
|
help="Output parquet: postcode + nested '{type} (by year)' columns",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--buffer-m",
|
"--output-records",
|
||||||
type=float,
|
type=Path,
|
||||||
default=DEFAULT_BUFFER_M,
|
required=True,
|
||||||
help="Outward buffer (metres) added to each postcode boundary",
|
help="Output parquet: one row per counted incident (last 7 years), postcode-sorted",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--max-postcodes",
|
"--max-postcodes",
|
||||||
|
|
@ -705,15 +835,12 @@ def main() -> None:
|
||||||
)
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.buffer_m <= 0:
|
|
||||||
raise SystemExit("--buffer-m must be greater than zero")
|
|
||||||
|
|
||||||
transform_crime_spatial(
|
transform_crime_spatial(
|
||||||
crime_dir=args.input,
|
crime_dir=args.input,
|
||||||
boundaries_dir=args.boundaries,
|
boundaries_dir=args.boundaries,
|
||||||
output_path=args.output,
|
output_path=args.output,
|
||||||
by_year_output_path=args.output_by_year,
|
by_year_output_path=args.output_by_year,
|
||||||
buffer_m=args.buffer_m,
|
records_output_path=args.output_records,
|
||||||
max_postcodes=args.max_postcodes,
|
max_postcodes=args.max_postcodes,
|
||||||
max_files=args.max_files,
|
max_files=args.max_files,
|
||||||
min_bar_months=args.min_bar_months,
|
min_bar_months=args.min_bar_months,
|
||||||
|
|
|
||||||
149
pipeline/transform/join_price_estimates.py
Normal file
149
pipeline/transform/join_price_estimates.py
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
"""Join the slim price estimates back onto properties.parquet.
|
||||||
|
|
||||||
|
Price estimation runs on ``price_inputs.parquet`` (built by ``property_base``
|
||||||
|
straight from epc_pp + arcgis, independently of merge's area features) and emits
|
||||||
|
``price_estimates.parquet`` — the natural key (Postcode + coalesced address) plus
|
||||||
|
``Estimated current price`` / ``Est. price per sqm``. This step joins those two
|
||||||
|
columns onto properties.parquet to produce the file the server consumes.
|
||||||
|
|
||||||
|
Why the natural key
|
||||||
|
-------------------
|
||||||
|
Estimates and properties are built by separate runs, so a positional row index
|
||||||
|
would not line up. Instead both derive the key ``(Postcode, coalesce(register
|
||||||
|
address, EPC address))`` — which is unique and non-null on the deduped dwelling
|
||||||
|
universe (see ``property_base._dedupe_collapsed_properties``) and identical on
|
||||||
|
both sides because both start from that same universe. So estimates map onto
|
||||||
|
properties 1:1 regardless of row order.
|
||||||
|
|
||||||
|
Re-running is safe: any pre-existing estimate columns are dropped first, and the
|
||||||
|
join is keyed (not positional), so a second run reproduces the same result. The
|
||||||
|
join refuses if any property has no estimate (the dwelling universes diverged,
|
||||||
|
e.g. a stale price_inputs vs a newer epc_pp) rather than silently leaving prices
|
||||||
|
null. Output is written to a temp file and atomically renamed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
from pipeline.transform.price_estimation.utils import (
|
||||||
|
ESTIMATE_COLUMNS,
|
||||||
|
JOIN_ADDRESS,
|
||||||
|
JOIN_KEYS,
|
||||||
|
join_address_expr,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def join_estimates(properties: Path, estimates_path: Path) -> int:
|
||||||
|
"""Augment ``properties`` in place with the estimate columns; return rows.
|
||||||
|
|
||||||
|
Joins the slim estimates onto properties by the natural key and atomically
|
||||||
|
replaces properties.parquet. Idempotent: any estimate columns already on the
|
||||||
|
file are dropped first.
|
||||||
|
"""
|
||||||
|
estimates = pl.scan_parquet(estimates_path)
|
||||||
|
est_cols = estimates.collect_schema().names()
|
||||||
|
missing = [c for c in (*JOIN_KEYS, *ESTIMATE_COLUMNS) if c not in est_cols]
|
||||||
|
if missing:
|
||||||
|
raise ValueError(f"{estimates_path}: missing columns {missing}")
|
||||||
|
|
||||||
|
stats = estimates.select(
|
||||||
|
n=pl.len(), unique=pl.struct(JOIN_KEYS).n_unique()
|
||||||
|
).collect(engine="streaming")
|
||||||
|
n_estimates, n_unique = stats["n"][0], stats["unique"][0]
|
||||||
|
if n_unique != n_estimates:
|
||||||
|
raise ValueError(
|
||||||
|
f"{estimates_path}: natural key {JOIN_KEYS} is not unique "
|
||||||
|
f"({n_estimates - n_unique:,} duplicate rows)"
|
||||||
|
)
|
||||||
|
|
||||||
|
n_properties = pl.scan_parquet(properties).select(pl.len()).collect().item()
|
||||||
|
|
||||||
|
# Drop any estimate columns already present (idempotent re-run) and attach the
|
||||||
|
# coalesced-address half of the natural key.
|
||||||
|
properties_keyed = (
|
||||||
|
pl.scan_parquet(properties)
|
||||||
|
.drop(ESTIMATE_COLUMNS, strict=False)
|
||||||
|
.with_columns(join_address_expr())
|
||||||
|
)
|
||||||
|
|
||||||
|
# Every property must have an estimate: estimates and properties come from the
|
||||||
|
# same dwelling universe, so a gap means a stale/foreign price_inputs (e.g.
|
||||||
|
# built from a different epc_pp). Fail loudly instead of nulling prices.
|
||||||
|
#
|
||||||
|
# This assumes properties.parquet contains ONLY epc_pp-derived dwellings, which
|
||||||
|
# is true for the production merge output. Running merge with --actual-listings
|
||||||
|
# appends listing seed rows whose (Postcode, address) keys are absent from
|
||||||
|
# price_inputs (built straight from epc_pp), which would trip the guard below.
|
||||||
|
# Enabling listing integration on the primary output therefore requires
|
||||||
|
# price_inputs to include those seed rows too.
|
||||||
|
unmatched = (
|
||||||
|
properties_keyed.select(JOIN_KEYS)
|
||||||
|
.join(estimates.select(JOIN_KEYS), on=JOIN_KEYS, how="anti")
|
||||||
|
.select(pl.len())
|
||||||
|
.collect(engine="streaming")
|
||||||
|
.item()
|
||||||
|
)
|
||||||
|
if unmatched:
|
||||||
|
raise ValueError(
|
||||||
|
f"{properties}: {unmatched:,} of {n_properties:,} properties have no "
|
||||||
|
"matching estimate; the price_inputs and properties dwelling universes "
|
||||||
|
"differ (regenerate price_inputs.parquet from the current epc_pp)."
|
||||||
|
)
|
||||||
|
|
||||||
|
# maintain_order="left" keeps properties in merge's row order; the unique key
|
||||||
|
# cannot fan the join out, so the row count is preserved.
|
||||||
|
result = properties_keyed.join(
|
||||||
|
estimates, on=JOIN_KEYS, how="left", maintain_order="left"
|
||||||
|
).drop(JOIN_ADDRESS)
|
||||||
|
|
||||||
|
tmp = properties.with_name(properties.name + ".tmp")
|
||||||
|
result.sink_parquet(tmp)
|
||||||
|
|
||||||
|
written = pl.scan_parquet(tmp).select(pl.len()).collect().item()
|
||||||
|
if written != n_properties:
|
||||||
|
tmp.unlink(missing_ok=True)
|
||||||
|
raise ValueError(
|
||||||
|
f"{properties}: join changed the row count "
|
||||||
|
f"({n_properties:,} -> {written:,})"
|
||||||
|
)
|
||||||
|
|
||||||
|
tmp.replace(properties)
|
||||||
|
return written
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Join price_estimates.parquet onto properties.parquet"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--properties",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="properties.parquet (read, then overwritten with the estimate "
|
||||||
|
"columns joined in)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--estimates",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Slim price_estimates.parquet from price_estimation.estimate",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
written = join_estimates(args.properties, args.estimates)
|
||||||
|
size_mb = args.properties.stat().st_size / (1024 * 1024)
|
||||||
|
n_priced = (
|
||||||
|
pl.scan_parquet(args.properties)
|
||||||
|
.filter(pl.col("Estimated current price").is_not_null())
|
||||||
|
.select(pl.len())
|
||||||
|
.collect()
|
||||||
|
.item()
|
||||||
|
)
|
||||||
|
print(f"Wrote {args.properties} ({size_mb:.1f} MB)")
|
||||||
|
print(f" {written:,} rows, {n_priced:,} with an estimated current price")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -29,10 +29,16 @@ from pipeline.utils.fuzzy_join import (
|
||||||
normalize_address_key,
|
normalize_address_key,
|
||||||
normalize_postcode_key,
|
normalize_postcode_key,
|
||||||
)
|
)
|
||||||
|
from pipeline.transform.property_base import (
|
||||||
|
MIN_FLOOR_AREA_M2,
|
||||||
|
_active_english_postcode_area,
|
||||||
|
_filter_to_active_english_postcodes,
|
||||||
|
build_property_base,
|
||||||
|
property_type_expr,
|
||||||
|
)
|
||||||
from pipeline.utils.normalize import drop_digit_tokens
|
from pipeline.utils.normalize import drop_digit_tokens
|
||||||
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
||||||
|
|
||||||
MIN_FLOOR_AREA_M2 = 10
|
|
||||||
CONSERVATION_AREA_FEATURE = "Within conservation area"
|
CONSERVATION_AREA_FEATURE = "Within conservation area"
|
||||||
# Named "Tree canopy" (not "Street tree") because the underlying density unions
|
# Named "Tree canopy" (not "Street tree") because the underlying density unions
|
||||||
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
|
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
|
||||||
|
|
@ -77,23 +83,42 @@ _AREA_COLUMNS = [
|
||||||
"% Mixed",
|
"% Mixed",
|
||||||
"% White",
|
"% White",
|
||||||
"% Other",
|
"% Other",
|
||||||
# Crime
|
# Crime — average annual recorded incident count (incidents/yr), 7-year and
|
||||||
"Anti-social behaviour (avg/yr)",
|
# 2-year windows. These are the filterable crime features; the per-incident
|
||||||
"Violence and sexual offences (avg/yr)",
|
# records live in a separate side table the server loads directly (it bypasses
|
||||||
"Criminal damage and arson (avg/yr)",
|
# the merge).
|
||||||
"Burglary (avg/yr)",
|
"Anti-social behaviour (/yr, 7y)",
|
||||||
"Vehicle crime (avg/yr)",
|
"Anti-social behaviour (/yr, 2y)",
|
||||||
"Robbery (avg/yr)",
|
"Violence and sexual offences (/yr, 7y)",
|
||||||
"Other theft (avg/yr)",
|
"Violence and sexual offences (/yr, 2y)",
|
||||||
"Shoplifting (avg/yr)",
|
"Criminal damage and arson (/yr, 7y)",
|
||||||
"Drugs (avg/yr)",
|
"Criminal damage and arson (/yr, 2y)",
|
||||||
"Possession of weapons (avg/yr)",
|
"Burglary (/yr, 7y)",
|
||||||
"Public order (avg/yr)",
|
"Burglary (/yr, 2y)",
|
||||||
"Bicycle theft (avg/yr)",
|
"Vehicle crime (/yr, 7y)",
|
||||||
"Theft from the person (avg/yr)",
|
"Vehicle crime (/yr, 2y)",
|
||||||
"Other crime (avg/yr)",
|
"Robbery (/yr, 7y)",
|
||||||
"Serious crime (avg/yr)",
|
"Robbery (/yr, 2y)",
|
||||||
"Minor crime (avg/yr)",
|
"Other theft (/yr, 7y)",
|
||||||
|
"Other theft (/yr, 2y)",
|
||||||
|
"Shoplifting (/yr, 7y)",
|
||||||
|
"Shoplifting (/yr, 2y)",
|
||||||
|
"Drugs (/yr, 7y)",
|
||||||
|
"Drugs (/yr, 2y)",
|
||||||
|
"Possession of weapons (/yr, 7y)",
|
||||||
|
"Possession of weapons (/yr, 2y)",
|
||||||
|
"Public order (/yr, 7y)",
|
||||||
|
"Public order (/yr, 2y)",
|
||||||
|
"Bicycle theft (/yr, 7y)",
|
||||||
|
"Bicycle theft (/yr, 2y)",
|
||||||
|
"Theft from the person (/yr, 7y)",
|
||||||
|
"Theft from the person (/yr, 2y)",
|
||||||
|
"Other crime (/yr, 7y)",
|
||||||
|
"Other crime (/yr, 2y)",
|
||||||
|
"Serious crime (/yr, 7y)",
|
||||||
|
"Serious crime (/yr, 2y)",
|
||||||
|
"Minor crime (/yr, 7y)",
|
||||||
|
"Minor crime (/yr, 2y)",
|
||||||
# Amenities
|
# Amenities
|
||||||
"Number of restaurants within 2km",
|
"Number of restaurants within 2km",
|
||||||
"Number of grocery shops and supermarkets within 2km",
|
"Number of grocery shops and supermarkets within 2km",
|
||||||
|
|
@ -189,8 +214,6 @@ _FINAL_RENAME_COLUMNS = {
|
||||||
"outstanding_primary_catchments": "Outstanding primary school catchments",
|
"outstanding_primary_catchments": "Outstanding primary school catchments",
|
||||||
"outstanding_secondary_catchments": "Outstanding secondary school catchments",
|
"outstanding_secondary_catchments": "Outstanding secondary school catchments",
|
||||||
"max_download_speed": "Max available download speed (Mbps)",
|
"max_download_speed": "Max available download speed (Mbps)",
|
||||||
"serious_crime_avg_yr": "Serious crime (avg/yr)",
|
|
||||||
"minor_crime_avg_yr": "Minor crime (avg/yr)",
|
|
||||||
"mean_monthly_rent": "Estimated monthly rent",
|
"mean_monthly_rent": "Estimated monthly rent",
|
||||||
"floor_height": "Interior height (m)",
|
"floor_height": "Interior height (m)",
|
||||||
"was_council_house": "Former council house",
|
"was_council_house": "Former council house",
|
||||||
|
|
@ -822,78 +845,6 @@ def _validate_property_postcodes(df: pl.DataFrame) -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
|
|
||||||
"""Return the supported postcode universe with geography join keys."""
|
|
||||||
return (
|
|
||||||
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
|
|
||||||
.filter(pl.col("doterm").is_null())
|
|
||||||
.select(
|
|
||||||
pl.col("pcds").alias("postcode"),
|
|
||||||
"lat",
|
|
||||||
pl.col("long").alias("lon"),
|
|
||||||
"ctry25cd",
|
|
||||||
pl.col("lsoa21cd").alias("lsoa21"),
|
|
||||||
pl.col("oa21cd").alias("oa21"),
|
|
||||||
pl.col("pcon24cd").alias("pcon"),
|
|
||||||
)
|
|
||||||
.drop_nulls(["postcode"])
|
|
||||||
.unique(["postcode"])
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _remap_terminated_postcodes(
|
|
||||||
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
|
|
||||||
) -> pl.LazyFrame:
|
|
||||||
return (
|
|
||||||
wide.join(
|
|
||||||
postcode_mapping,
|
|
||||||
left_on="postcode",
|
|
||||||
right_on="old_postcode",
|
|
||||||
how="left",
|
|
||||||
)
|
|
||||||
.with_columns(
|
|
||||||
pl.coalesce("new_postcode", "postcode").alias("postcode"),
|
|
||||||
)
|
|
||||||
.drop("new_postcode")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
|
|
||||||
"""Keep one row per (postcode, address) — the most-recent transaction.
|
|
||||||
|
|
||||||
The terminated-postcode remap can map two distinct postcodes onto one active
|
|
||||||
successor, collapsing the same physical address onto a single
|
|
||||||
(postcode, address) key with conflicting sale records. Keep the row with the
|
|
||||||
latest date_of_transfer so the headline price/date reflect the most recent
|
|
||||||
transaction; genuinely distinct addresses are untouched.
|
|
||||||
|
|
||||||
The dedup key coalesces the price-paid address with the EPC address: EPC-only
|
|
||||||
dwellings (never sold) have a null pp_address, so keying on pp_address alone
|
|
||||||
would collapse EVERY EPC-only dwelling in a postcode onto one
|
|
||||||
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
|
|
||||||
address is unique within its postcode (the EPC frame is deduped on
|
|
||||||
address+postcode upstream), so the coalesced key keeps them distinct while
|
|
||||||
leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
|
|
||||||
a sale exists.
|
|
||||||
"""
|
|
||||||
return (
|
|
||||||
wide.with_columns(
|
|
||||||
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
|
|
||||||
)
|
|
||||||
.sort("date_of_transfer", descending=True, nulls_last=True)
|
|
||||||
.unique(
|
|
||||||
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
|
|
||||||
)
|
|
||||||
.drop("_dedupe_address")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _filter_to_active_english_postcodes(
|
|
||||||
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
|
|
||||||
) -> pl.LazyFrame:
|
|
||||||
return wide.join(active_postcodes, on="postcode", how="semi")
|
|
||||||
|
|
||||||
|
|
||||||
def _join_area_side_tables(
|
def _join_area_side_tables(
|
||||||
base: pl.LazyFrame,
|
base: pl.LazyFrame,
|
||||||
*,
|
*,
|
||||||
|
|
@ -923,21 +874,15 @@ def _join_area_side_tables(
|
||||||
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
|
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
|
||||||
base = base.join(tenure, on="lsoa21", how="left")
|
base = base.join(tenure, on="lsoa21", how="left")
|
||||||
|
|
||||||
# Crime is counted spatially per postcode (incidents within 50m of the
|
# Crime is counted spatially per postcode (incidents within the boundary
|
||||||
# postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
|
# buffer), so it joins on postcode rather than LSOA. crime_spatial writes
|
||||||
# precomputes the Serious/Minor headline rollups as the mean of the by-year
|
# average-annual-count columns ("{type} (/yr, 7y|2y)"), including the
|
||||||
# rollup bars; read those straight through (renamed to the internal columns
|
# Serious/Minor rollups (the exact sum of their components); all pass straight
|
||||||
# _finalize_merged_columns expects) rather than re-summing the per-type
|
# through to display/filtering. A postcode absent from the crime table keeps
|
||||||
# avg/yr columns — summing divides each type by its OWN years-present and
|
# null values via the left join (no fabricated zero). The per-incident records
|
||||||
# overstates the rollup when types differ in coverage. A postcode absent from
|
# are a separate side table the server loads directly, so it is not joined
|
||||||
# the crime table keeps null rollups via the left join (no fabricated zero);
|
# here.
|
||||||
# the per-type avg/yr columns pass through unchanged for display.
|
base = base.join(crime, on="postcode", how="left")
|
||||||
base = base.join(crime, on="postcode", how="left").rename(
|
|
||||||
{
|
|
||||||
"Serious crime (avg/yr)": "serious_crime_avg_yr",
|
|
||||||
"Minor crime (avg/yr)": "minor_crime_avg_yr",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
base = base.join(median_age, on="lsoa21", how="left")
|
base = base.join(median_age, on="lsoa21", how="left")
|
||||||
base = base.join(election, on="pcon", how="left")
|
base = base.join(election, on="pcon", how="left")
|
||||||
|
|
@ -2386,27 +2331,17 @@ def _build(
|
||||||
)
|
)
|
||||||
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
||||||
|
|
||||||
wide = pl.scan_parquet(epc_pp_path).filter(
|
# The dwelling universe — floor filter, terminated-postcode remap,
|
||||||
pl.col("total_floor_area").is_null()
|
# collapse-dedupe, restrict to active English postcodes — is shared with
|
||||||
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
|
# price estimation so estimates line up 1:1 with these rows. See
|
||||||
)
|
# pipeline.transform.property_base.
|
||||||
|
wide = build_property_base(epc_pp_path, arcgis_path)
|
||||||
# Remap terminated postcodes to nearest active successor before filtering to
|
|
||||||
# the supported active-English postcode universe. Historical properties from
|
|
||||||
# terminated English postcodes are retained under their successor postcode.
|
|
||||||
postcode_mapping = build_postcode_mapping(arcgis_path)
|
|
||||||
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
|
|
||||||
# The remap can collapse two terminated postcodes onto one active successor,
|
|
||||||
# duplicating a physical address's (postcode, pp_address) key; keep only the
|
|
||||||
# most-recent transaction per address before the per-postcode joins.
|
|
||||||
wide = _dedupe_collapsed_properties(wide)
|
|
||||||
arcgis_raw = pl.scan_parquet(arcgis_path)
|
arcgis_raw = pl.scan_parquet(arcgis_path)
|
||||||
arcgis = _active_english_postcode_area(arcgis_raw)
|
arcgis = _active_english_postcode_area(arcgis_raw)
|
||||||
active_postcodes = arcgis.select("postcode").unique()
|
active_postcodes = arcgis.select("postcode").unique()
|
||||||
active_postcode_count = (
|
active_postcode_count = (
|
||||||
active_postcodes.select(pl.len()).collect(engine="streaming").item()
|
active_postcodes.select(pl.len()).collect(engine="streaming").item()
|
||||||
)
|
)
|
||||||
wide = _filter_to_active_english_postcodes(wide, active_postcodes)
|
|
||||||
|
|
||||||
if listed_buildings_path is not None:
|
if listed_buildings_path is not None:
|
||||||
active_postcodes_for_listed = (
|
active_postcodes_for_listed = (
|
||||||
|
|
@ -2542,37 +2477,10 @@ def _build(
|
||||||
how="left",
|
how="left",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Derive property_type: prefer EPC data, fall back to price-paid.
|
# Derive property_type (EPC preferred, price-paid fallback, built_form for
|
||||||
# For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer detail.
|
# houses). Shared with price_inputs so the estimate uses the same type; see
|
||||||
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
|
# property_base.property_type_expr.
|
||||||
["NO DATA!", "Not Recorded"]
|
wide = wide.with_columns(property_type_expr().alias("property_type"))
|
||||||
)
|
|
||||||
has_epc = pl.col("epc_property_type").is_not_null()
|
|
||||||
is_house = pl.col("epc_property_type") == "House"
|
|
||||||
wide = wide.with_columns(
|
|
||||||
pl.when(has_epc & is_house & ~bad_built_form)
|
|
||||||
.then(pl.col("built_form"))
|
|
||||||
.when(has_epc & is_house)
|
|
||||||
.then(pl.col("pp_property_type"))
|
|
||||||
.when(has_epc)
|
|
||||||
.then(pl.col("epc_property_type"))
|
|
||||||
.otherwise(pl.col("pp_property_type"))
|
|
||||||
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
|
|
||||||
# collapse terrace sub-types, and fold rare types into "Other"
|
|
||||||
.replace(
|
|
||||||
{
|
|
||||||
"Flat": "Flats/Maisonettes",
|
|
||||||
"Maisonette": "Flats/Maisonettes",
|
|
||||||
"End-Terrace": "Terraced",
|
|
||||||
"Mid-Terrace": "Terraced",
|
|
||||||
"Enclosed End-Terrace": "Terraced",
|
|
||||||
"Enclosed Mid-Terrace": "Terraced",
|
|
||||||
"Bungalow": "Other",
|
|
||||||
"Park home": "Other",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
.alias("property_type")
|
|
||||||
)
|
|
||||||
|
|
||||||
wide = wide.with_columns(
|
wide = wide.with_columns(
|
||||||
pl.when(pl.col("duration") == "U")
|
pl.when(pl.col("duration") == "U")
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,9 @@ The output of `process_oa` is `list[(postcode, polygon)]` — the per-OA fragmen
|
||||||
|
|
||||||
**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 3000–5000 m² building blocks) before this change.
|
**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 3000–5000 m² building blocks) before this change.
|
||||||
|
|
||||||
**GeoJSON output** (`output.py:write_district_geojson`): Two passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
|
**Greenspace subtraction is connectivity-preserving** (`greenspace.py:subtract_greenspace`): park/water polygons are subtracted from each postcode, but greenspace that *crosses* a postcode (a river, a strip of parkland, a golf course through a village) would otherwise split it into scattered pieces. When the subtraction disconnects a postcode, `_reconnect_split` re-adds the narrowest removed necks — a morphological closing (`_RECONNECT_BRIDGE_M`, 25 m) clipped to the original postcode footprint — so parts ≤ ~50 m apart stay joined by a thin bridge of the postcode's own land (no address moves); genuinely wide barriers stay subtracted and the postcode legitimately splits.
|
||||||
|
|
||||||
|
**GeoJSON output** (`output.py:write_district_geojson`): three passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 **de-fragments** the partition (`_eliminate_small_detached_parts`): a detached part that is *both* small in absolute terms (< `_ELIM_ABS_MAX_M2`, 3000 m²) *and* a minor fraction (< `_ELIM_FRAC_MAX`, 15%) of its postcode is absorbed into the neighbouring postcode it shares the most boundary with — the classic GIS *eliminate*. This removes the Voronoi/INSPIRE/seam *scatter* that left ~1/3 of postcodes non-contiguous, while a genuine bisection (two substantial parts split by a river/railway) keeps both parts. The land is **reassigned**, never dropped, so the output stays a gapless partition and coverage is conserved; the largest part of every postcode is always retained, so no active postcode is dropped (a tiny neighbour-less sliver in removed greenspace is dropped, a larger isolated patch is kept). Pass 3 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
|
||||||
|
|
||||||
## Memory architecture
|
## Memory architecture
|
||||||
|
|
||||||
|
|
@ -107,6 +109,7 @@ Key design choices:
|
||||||
2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
|
2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
|
||||||
3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
|
3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
|
||||||
4. **A postcode split across an OA seam keeps all its substantial parts** — `merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
|
4. **A postcode split across an OA seam keeps all its substantial parts** — `merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
|
||||||
|
5. **Postcodes are contiguous unless genuinely split** — most non-contiguity is *scatter* (a unit drawn as many disconnected specks) from point-Voronoi over sparse/interleaved UPRNs, greenspace cutting across a unit, and overlap/seam slivers. Connectivity-preserving greenspace subtraction + the `_eliminate_small_detached_parts` de-fragmentation pass absorb that scatter into neighbours (coverage-conserving), cutting the share of multi-part postcodes roughly in half (~30% → ~14% measured on the worst rural/coastal districts) without dropping any postcode or leaving coverage gaps. Genuine bisections (river/railway/major road, or a detached part above the absolute+fraction thresholds) are preserved.
|
||||||
|
|
||||||
## Module structure
|
## Module structure
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from shapely import make_valid, wkb
|
||||||
from shapely.geometry import MultiPolygon, Polygon
|
from shapely.geometry import MultiPolygon, Polygon
|
||||||
from shapely.strtree import STRtree
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
from .geometry import safe_difference, safe_union
|
from .geometry import _SNAP_GRID, _poly_valid, safe_difference, safe_intersection, safe_union
|
||||||
|
|
||||||
|
|
||||||
def load_greenspace(path: Path) -> tuple[STRtree, list]:
|
def load_greenspace(path: Path) -> tuple[STRtree, list]:
|
||||||
|
|
@ -36,6 +36,42 @@ def load_greenspace(path: Path) -> tuple[STRtree, list]:
|
||||||
|
|
||||||
MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed
|
MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed
|
||||||
|
|
||||||
|
# Greenspace that merely trims the edge of a postcode is fine, but greenspace
|
||||||
|
# that CROSSES it (a river, a strip of parkland, a golf course running through a
|
||||||
|
# village) splits the postcode into a MultiPolygon -- one the map then draws as
|
||||||
|
# several disconnected pieces. When subtraction disconnects a postcode, re-add
|
||||||
|
# the postcode's OWN removed land along the narrowest necks (a morphological
|
||||||
|
# closing clipped to the original footprint) so the parts stay joined by a thin
|
||||||
|
# bridge. Parts left more than ~2x this width apart (a genuinely wide barrier)
|
||||||
|
# stay split. Because the bridge is the postcode's own land, no address moves and
|
||||||
|
# only a thin sliver of green is kept back.
|
||||||
|
_RECONNECT_BRIDGE_M = 25.0
|
||||||
|
|
||||||
|
|
||||||
|
def _reconnect_split(
|
||||||
|
result: Polygon | MultiPolygon, postcode_geom: Polygon | MultiPolygon
|
||||||
|
) -> Polygon | MultiPolygon:
|
||||||
|
"""Re-join postcode parts that greenspace subtraction pulled apart by re-adding
|
||||||
|
the narrow removed necks (within the original postcode), leaving wide barriers
|
||||||
|
intact."""
|
||||||
|
if result.geom_type != "MultiPolygon":
|
||||||
|
return result
|
||||||
|
closed = result.buffer(_RECONNECT_BRIDGE_M).buffer(-_RECONNECT_BRIDGE_M)
|
||||||
|
if not closed.is_valid:
|
||||||
|
closed = make_valid(closed)
|
||||||
|
# The closing material that lies inside the original postcode but outside the
|
||||||
|
# subtraction result == the thin green necks linking the parts. The exact
|
||||||
|
# overlay path can leave line/point debris (coincident edges) that is
|
||||||
|
# zero-area but NOT is_empty; `_poly_valid` strips it to polygons only, so the
|
||||||
|
# is_empty guard works and the union can never return a GeometryCollection
|
||||||
|
# (which `to_wgs84_geojson_multi` would silently truncate to a single part).
|
||||||
|
bridges = _poly_valid(
|
||||||
|
safe_difference(safe_intersection(closed, postcode_geom), result), _SNAP_GRID
|
||||||
|
)
|
||||||
|
if bridges.is_empty:
|
||||||
|
return result
|
||||||
|
return _poly_valid(safe_union([result, bridges]), _SNAP_GRID)
|
||||||
|
|
||||||
|
|
||||||
def subtract_greenspace(
|
def subtract_greenspace(
|
||||||
postcode_geom: Polygon | MultiPolygon,
|
postcode_geom: Polygon | MultiPolygon,
|
||||||
|
|
@ -48,6 +84,10 @@ def subtract_greenspace(
|
||||||
of intersecting greenspace from the postcode polygon. If subtraction
|
of intersecting greenspace from the postcode polygon. If subtraction
|
||||||
would remove >90% of the area, keeps the original (the postcode
|
would remove >90% of the area, keeps the original (the postcode
|
||||||
genuinely covers that land, e.g. churchyards, riverside addresses).
|
genuinely covers that land, e.g. churchyards, riverside addresses).
|
||||||
|
|
||||||
|
If the subtraction disconnects the postcode (greenspace crossing it),
|
||||||
|
:func:`_reconnect_split` re-adds the narrowest removed necks so the postcode
|
||||||
|
stays a single piece rather than shipping as scattered fragments.
|
||||||
"""
|
"""
|
||||||
candidate_idxs = tree.query(postcode_geom)
|
candidate_idxs = tree.query(postcode_geom)
|
||||||
if len(candidate_idxs) == 0:
|
if len(candidate_idxs) == 0:
|
||||||
|
|
@ -74,4 +114,4 @@ def subtract_greenspace(
|
||||||
if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION):
|
if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION):
|
||||||
return postcode_geom
|
return postcode_geom
|
||||||
|
|
||||||
return result
|
return _reconnect_split(result, postcode_geom)
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import shutil
|
import shutil
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -383,6 +384,164 @@ def _resolve_overlaps(
|
||||||
return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
|
return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
|
||||||
|
|
||||||
|
|
||||||
|
# A detached (non-largest) part of a postcode is absorbed into the neighbouring
|
||||||
|
# postcode it shares the most boundary with, when the part is BOTH small in
|
||||||
|
# absolute terms AND a minor fraction of its postcode. This removes the
|
||||||
|
# Voronoi/INSPIRE/overlap-seam SCATTER that otherwise left ~1/3 of postcodes
|
||||||
|
# non-contiguous (a single unit drawn as many disconnected specks), while keeping
|
||||||
|
# genuine splits: a postcode bisected by a river/railway into two SUBSTANTIAL
|
||||||
|
# parts has both parts above these thresholds and is left alone. The land is
|
||||||
|
# REASSIGNED to an adjacent postcode wherever a neighbour exists, so coverage is
|
||||||
|
# conserved and the output stays a partition; only a tiny neighbour-less sliver
|
||||||
|
# (< _ELIM_DROP_BELOW_M2, snap/overlap debris floating in removed greenspace) is
|
||||||
|
# dropped, and a larger isolated part (a genuine detached hamlet) is kept. The
|
||||||
|
# largest part of every postcode is always retained, so no postcode is dropped.
|
||||||
|
_ELIM_ABS_MAX_M2 = 3000.0 # absolute size below which a minor part reads as scatter
|
||||||
|
_ELIM_FRAC_MAX = 0.15 # ...and only when it is < this fraction of the postcode
|
||||||
|
_ELIM_DROP_BELOW_M2 = 200.0 # a neighbour-less sliver this small is snap debris -> drop
|
||||||
|
# WGS84-degree distances at UK latitudes (1e-6 deg ~ 0.11 m at ~53N)
|
||||||
|
_ELIM_SHARED_EPS_DEG = 5e-7 # ~0.05 m: thin probe whose overlap area ~ shared-edge length
|
||||||
|
# Candidate-gather + nearest-neighbour fallback radius, in degrees so it needs no
|
||||||
|
# per-part reprojection. The metric reach is mildly anisotropic (~2.2 m N-S,
|
||||||
|
# ~1.3-1.4 m E-W at England's latitudes); this only bounds the rare gapped-seam
|
||||||
|
# fallback (the dominant border-sharing path is unaffected), so the approximation
|
||||||
|
# is harmless. Gather and accept use the SAME radius, so there is no dead band.
|
||||||
|
_ELIM_NEAREST_MAX_DEG = 2e-5
|
||||||
|
_M_PER_DEG_LAT = 111_320.0
|
||||||
|
_ELIM_ITERATIONS = 2
|
||||||
|
|
||||||
|
|
||||||
|
def _approx_area_m2(geom_deg) -> float:
|
||||||
|
"""Metric area of a WGS84 polygon via a latitude-scaled planar approximation.
|
||||||
|
|
||||||
|
Accurate to a few percent at the few-hundred-to-few-thousand m^2 scale these
|
||||||
|
thresholds work at, and far cheaper than a per-part CRS transform over the
|
||||||
|
full ~1.8M postcodes.
|
||||||
|
"""
|
||||||
|
area_deg2 = geom_deg.area
|
||||||
|
if area_deg2 == 0.0:
|
||||||
|
return 0.0
|
||||||
|
centroid = geom_deg.centroid
|
||||||
|
if centroid.is_empty:
|
||||||
|
lat = (geom_deg.bounds[1] + geom_deg.bounds[3]) / 2
|
||||||
|
else:
|
||||||
|
lat = centroid.y
|
||||||
|
return area_deg2 * _M_PER_DEG_LAT * _M_PER_DEG_LAT * math.cos(math.radians(lat))
|
||||||
|
|
||||||
|
|
||||||
|
def _geom_parts(geom):
|
||||||
|
return list(geom.geoms) if geom.geom_type == "MultiPolygon" else [geom]
|
||||||
|
|
||||||
|
|
||||||
|
def _eliminate_small_detached_parts(
|
||||||
|
items: list[tuple[str, Polygon | MultiPolygon]],
|
||||||
|
) -> list[tuple[str, Polygon | MultiPolygon]]:
|
||||||
|
"""De-fragment the partition: absorb small detached parts into the neighbour
|
||||||
|
they border most (see the module note above).
|
||||||
|
|
||||||
|
Runs on the de-overlapped WGS84 geometries as the last shaping step. The
|
||||||
|
largest part of every postcode is always retained, so no active postcode is
|
||||||
|
dropped; parts are moved between postcodes (or, for tiny neighbour-less
|
||||||
|
slivers, dropped), so total covered area is conserved to within rounding.
|
||||||
|
"""
|
||||||
|
geoms: dict[str, Polygon | MultiPolygon] = {pc: g for pc, g in items}
|
||||||
|
for _ in range(_ELIM_ITERATIONS):
|
||||||
|
recs: list[tuple[str, Polygon, float]] = []
|
||||||
|
total: dict[str, float] = defaultdict(float)
|
||||||
|
largest: dict[str, float] = defaultdict(float)
|
||||||
|
for pc, geom in geoms.items():
|
||||||
|
for part in _geom_parts(geom):
|
||||||
|
if part.is_empty:
|
||||||
|
continue
|
||||||
|
area = _approx_area_m2(part)
|
||||||
|
recs.append((pc, part, area))
|
||||||
|
total[pc] += area
|
||||||
|
if area > largest[pc]:
|
||||||
|
largest[pc] = area
|
||||||
|
|
||||||
|
tree = STRtree([part for _, part, _ in recs])
|
||||||
|
assign: dict[int, str | None] = {}
|
||||||
|
for i, (pc, part, area) in enumerate(recs):
|
||||||
|
if area >= largest[pc]:
|
||||||
|
continue # the largest part always stays -> a postcode never vanishes
|
||||||
|
if not (area < _ELIM_ABS_MAX_M2 and area < _ELIM_FRAC_MAX * total[pc]):
|
||||||
|
continue # substantial or major-fraction part = genuine split, keep
|
||||||
|
|
||||||
|
best_pc = None
|
||||||
|
best_score = 0.0
|
||||||
|
nearest_pc = None
|
||||||
|
nearest_d = float("inf")
|
||||||
|
# Gather candidates out to the nearest-fallback radius (not just the
|
||||||
|
# smaller border-probe radius), so a gapped seam in the
|
||||||
|
# (border, nearest] band can actually be reassigned rather than dropped.
|
||||||
|
for j in tree.query(part.buffer(_ELIM_NEAREST_MAX_DEG)):
|
||||||
|
if j == i:
|
||||||
|
continue
|
||||||
|
other_pc, other_geom, _ = recs[j]
|
||||||
|
if other_pc == pc:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
score = part.buffer(_ELIM_SHARED_EPS_DEG).intersection(other_geom).area
|
||||||
|
except GEOSException:
|
||||||
|
score = 0.0
|
||||||
|
# Ties broken by the lexicographically smaller postcode so the
|
||||||
|
# result is independent of STRtree traversal order (matches the
|
||||||
|
# stable tie-break in _resolve_overlaps; keeps output byte-stable
|
||||||
|
# across shapely/GEOS upgrades for content-hash caching).
|
||||||
|
if score > best_score or (
|
||||||
|
score == best_score and best_pc is not None and other_pc < best_pc
|
||||||
|
):
|
||||||
|
best_score = score
|
||||||
|
best_pc = other_pc
|
||||||
|
dist = part.distance(other_geom)
|
||||||
|
if dist < nearest_d or (
|
||||||
|
dist == nearest_d
|
||||||
|
and nearest_pc is not None
|
||||||
|
and other_pc < nearest_pc
|
||||||
|
):
|
||||||
|
nearest_d = dist
|
||||||
|
nearest_pc = other_pc
|
||||||
|
|
||||||
|
if best_pc is not None and best_score > 0:
|
||||||
|
assign[i] = best_pc # shares a border -> absorb into that neighbour
|
||||||
|
elif nearest_pc is not None and nearest_d <= _ELIM_NEAREST_MAX_DEG:
|
||||||
|
assign[i] = nearest_pc # gapped seam -> nearest neighbour
|
||||||
|
elif area < _ELIM_DROP_BELOW_M2:
|
||||||
|
assign[i] = None # neighbour-less snap/overlap sliver -> drop
|
||||||
|
# else: keep with its own postcode (a genuine isolated patch)
|
||||||
|
|
||||||
|
if not assign:
|
||||||
|
break
|
||||||
|
|
||||||
|
new_parts: dict[str, list] = defaultdict(list)
|
||||||
|
for i, (pc, part, _) in enumerate(recs):
|
||||||
|
if i in assign:
|
||||||
|
target = assign[i]
|
||||||
|
if target is None:
|
||||||
|
continue
|
||||||
|
new_parts[target].append(part)
|
||||||
|
else:
|
||||||
|
new_parts[pc].append(part)
|
||||||
|
|
||||||
|
rebuilt: dict[str, Polygon | MultiPolygon] = {}
|
||||||
|
for pc, parts in new_parts.items():
|
||||||
|
if len(parts) == 1:
|
||||||
|
rebuilt[pc] = parts[0]
|
||||||
|
else:
|
||||||
|
merged = safe_union(parts, grid=_OUTPUT_PRECISION_DEG)
|
||||||
|
if not merged.is_empty:
|
||||||
|
rebuilt[pc] = merged
|
||||||
|
# Carry forward any postcode that contributed no parts to recs (e.g. an
|
||||||
|
# all-empty input geometry, which the part loop skips): never drop a
|
||||||
|
# postcode just because reassignment fired elsewhere in this pass.
|
||||||
|
for pc, geom in geoms.items():
|
||||||
|
if pc not in rebuilt:
|
||||||
|
rebuilt[pc] = geom
|
||||||
|
geoms = rebuilt
|
||||||
|
|
||||||
|
return list(geoms.items())
|
||||||
|
|
||||||
|
|
||||||
def _round_coords(coords, ndigits=6):
|
def _round_coords(coords, ndigits=6):
|
||||||
if coords and isinstance(coords[0], (int, float)):
|
if coords and isinstance(coords[0], (int, float)):
|
||||||
return [round(coords[0], ndigits), round(coords[1], ndigits)]
|
return [round(coords[0], ndigits), round(coords[1], ndigits)]
|
||||||
|
|
@ -501,6 +660,11 @@ def write_district_geojson(
|
||||||
# Remove overlap strips so the output is a clean partition.
|
# Remove overlap strips so the output is a clean partition.
|
||||||
converted = _resolve_overlaps(converted)
|
converted = _resolve_overlaps(converted)
|
||||||
|
|
||||||
|
# De-fragment: absorb small detached parts (Voronoi/INSPIRE/seam scatter) into
|
||||||
|
# the neighbour they border most, so postcodes stop shipping as many
|
||||||
|
# disconnected specks. Coverage-preserving; runs on the final partition.
|
||||||
|
converted = _eliminate_small_detached_parts(converted)
|
||||||
|
|
||||||
by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
|
by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
|
||||||
for pc, geom in converted:
|
for pc, geom in converted:
|
||||||
parts = pc.split()
|
parts = pc.split()
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ Each test targets a specific bug or edge case identified during code review.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import polars as pl
|
import polars as pl
|
||||||
|
|
@ -21,6 +22,8 @@ from .inspire import build_inspire_index
|
||||||
from .oa_boundaries import parse_gpkg_geometry
|
from .oa_boundaries import parse_gpkg_geometry
|
||||||
from .greenspace import subtract_greenspace
|
from .greenspace import subtract_greenspace
|
||||||
from .output import (
|
from .output import (
|
||||||
|
_approx_area_m2,
|
||||||
|
_eliminate_small_detached_parts,
|
||||||
_fill_holes,
|
_fill_holes,
|
||||||
merge_fragments,
|
merge_fragments,
|
||||||
to_wgs84_geojson,
|
to_wgs84_geojson,
|
||||||
|
|
@ -1830,3 +1833,231 @@ class TestFragmentsCache:
|
||||||
cache.write_text("c")
|
cache.write_text("c")
|
||||||
# arcgis is optional/absent — it cannot have invalidated the cache.
|
# arcgis is optional/absent — it cannot have invalidated the cache.
|
||||||
assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True
|
assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# De-fragmentation: small detached parts are absorbed into their best neighbour
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
# WGS84 helper: build a box from METRE offsets around a fixed England anchor, so
|
||||||
|
# the eliminate pass (which works in WGS84 degrees and measures area with a
|
||||||
|
# latitude-scaled approximation) sees realistic coordinates and areas.
|
||||||
|
_ANCHOR_LON, _ANCHOR_LAT = -0.1, 51.5
|
||||||
|
_M_PER_DEG_LAT = 111_320.0
|
||||||
|
_M_PER_DEG_LON = 111_320.0 * math.cos(math.radians(_ANCHOR_LAT))
|
||||||
|
|
||||||
|
|
||||||
|
def _mbox(x0, y0, x1, y1):
|
||||||
|
return box(
|
||||||
|
_ANCHOR_LON + x0 / _M_PER_DEG_LON,
|
||||||
|
_ANCHOR_LAT + y0 / _M_PER_DEG_LAT,
|
||||||
|
_ANCHOR_LON + x1 / _M_PER_DEG_LON,
|
||||||
|
_ANCHOR_LAT + y1 / _M_PER_DEG_LAT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _as_dict(items):
|
||||||
|
return dict(items)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEliminateSmallDetachedParts:
|
||||||
|
"""A small detached part should be absorbed into the postcode it borders most,
|
||||||
|
de-fragmenting the unit while conserving coverage and never dropping a
|
||||||
|
postcode. Genuine large splits must survive."""
|
||||||
|
|
||||||
|
def test_small_island_absorbed_by_surrounding_neighbour(self):
|
||||||
|
# A: a big main blob (left) plus a small 30x30m island sitting INSIDE B's
|
||||||
|
# territory; B fills the middle/right with a hole where the island is.
|
||||||
|
main_a = _mbox(0, 0, 100, 200) # 20000 m²
|
||||||
|
island = _mbox(150, 90, 180, 120) # 900 m², surrounded by B
|
||||||
|
a = MultiPolygon([main_a, island])
|
||||||
|
b = _mbox(100, 0, 300, 200).difference(island) # hole around the island
|
||||||
|
before = unary_union([a, b]).area
|
||||||
|
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
|
||||||
|
|
||||||
|
assert "A" in out and "B" in out, "no postcode may be dropped"
|
||||||
|
assert out["A"].geom_type == "Polygon", "A's island should be absorbed away"
|
||||||
|
# The island area moves to B (the surrounding postcode); coverage is kept.
|
||||||
|
assert out["B"].contains(island.representative_point())
|
||||||
|
after = unary_union([out["A"], out["B"]]).area
|
||||||
|
assert after == pytest.approx(before, rel=1e-6), "coverage must be conserved"
|
||||||
|
|
||||||
|
def test_genuine_large_split_is_kept(self):
|
||||||
|
# Two substantial parts (both 40000 m²) far apart — a real bisection, not
|
||||||
|
# scatter. Neither is below the absolute/fraction thresholds, so both stay.
|
||||||
|
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(400, 0, 600, 200)])
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||||
|
assert out["A"].geom_type == "MultiPolygon"
|
||||||
|
assert len(out["A"].geoms) == 2
|
||||||
|
|
||||||
|
def test_midsize_minor_part_kept_when_above_abs_threshold(self):
|
||||||
|
# A big main plus a 10000 m² detached part: above the 3000 m² absolute
|
||||||
|
# threshold, so it is a genuine secondary piece and must NOT be eliminated,
|
||||||
|
# even though it is a small fraction of the postcode.
|
||||||
|
a = MultiPolygon([_mbox(0, 0, 300, 300), _mbox(1000, 0, 1100, 100)])
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||||
|
assert out["A"].geom_type == "MultiPolygon"
|
||||||
|
|
||||||
|
def test_tiny_neighbourless_sliver_is_dropped(self):
|
||||||
|
# A main blob plus a 10x10m (100 m²) sliver far from anything: below the
|
||||||
|
# drop threshold with no neighbour to absorb it -> dropped as snap debris.
|
||||||
|
a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(1000, 1000, 1010, 1010)])
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
|
||||||
|
assert "A" in out, "the postcode itself must survive"
|
||||||
|
assert out["A"].geom_type == "Polygon"
|
||||||
|
|
||||||
|
def test_largest_part_always_retained_no_postcode_dropped(self):
|
||||||
|
# Even a postcode that is ENTIRELY a tiny sliver keeps its (largest) part —
|
||||||
|
# active postcodes must never be dropped (validate_outputs is zero-tolerance).
|
||||||
|
tiny = _mbox(500, 500, 505, 505) # 25 m², the whole postcode
|
||||||
|
big = _mbox(0, 0, 300, 300)
|
||||||
|
out = _as_dict(
|
||||||
|
_eliminate_small_detached_parts([("TINY", tiny), ("BIG", big)])
|
||||||
|
)
|
||||||
|
assert "TINY" in out and not out["TINY"].is_empty
|
||||||
|
assert "BIG" in out
|
||||||
|
|
||||||
|
def test_no_overlaps_introduced(self):
|
||||||
|
# Absorbing parts must keep the output a partition (no double-coverage).
|
||||||
|
main_a = _mbox(0, 0, 100, 200)
|
||||||
|
island = _mbox(150, 90, 180, 120)
|
||||||
|
a = MultiPolygon([main_a, island])
|
||||||
|
b = _mbox(100, 0, 300, 200).difference(island)
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
|
||||||
|
overlap = out["A"].intersection(out["B"]).area
|
||||||
|
assert overlap < (1.0 / (_M_PER_DEG_LAT * _M_PER_DEG_LON)), "no overlap"
|
||||||
|
|
||||||
|
def test_empty_postcode_not_dropped_when_reassignment_fires(self):
|
||||||
|
# Regression: an empty-geometry postcode must survive even when a
|
||||||
|
# reassignment fires elsewhere in the same pass (the rebuild used to drop
|
||||||
|
# any key absent from `recs`). No active postcode may ever be dropped.
|
||||||
|
main_a = _mbox(0, 0, 100, 200)
|
||||||
|
island = _mbox(150, 90, 180, 120) # B absorbs this -> reassignment fires
|
||||||
|
a = MultiPolygon([main_a, island])
|
||||||
|
b = _mbox(100, 0, 300, 200).difference(island)
|
||||||
|
empty = Polygon() # degenerate input that contributes no parts
|
||||||
|
out = _as_dict(
|
||||||
|
_eliminate_small_detached_parts([("A", a), ("B", b), ("EMPTY", empty)])
|
||||||
|
)
|
||||||
|
assert "EMPTY" in out, "an empty-geometry postcode must not be dropped"
|
||||||
|
assert "A" in out and "B" in out
|
||||||
|
|
||||||
|
def test_tiebreak_is_deterministic_smaller_postcode_wins(self):
|
||||||
|
# An island bordered by TWO postcodes with EQUAL shared edges must go to
|
||||||
|
# the lexicographically smaller postcode, independent of STRtree order.
|
||||||
|
a_main = _mbox(0, 0, 100, 100) # A's main, far from the island
|
||||||
|
island = _mbox(300, 50, 320, 70) # 400 m², A's detached part
|
||||||
|
a = MultiPolygon([a_main, island])
|
||||||
|
bbb = _mbox(250, 0, 300, 120) # borders island's left edge
|
||||||
|
ccc = _mbox(320, 0, 370, 120) # borders island's right edge (equal)
|
||||||
|
out = _as_dict(
|
||||||
|
_eliminate_small_detached_parts([("A", a), ("BBB", bbb), ("CCC", ccc)])
|
||||||
|
)
|
||||||
|
pt = island.representative_point()
|
||||||
|
assert out["BBB"].contains(pt), "tie must go to the smaller postcode string"
|
||||||
|
assert not out["CCC"].contains(pt)
|
||||||
|
assert out["A"].geom_type == "Polygon"
|
||||||
|
|
||||||
|
def test_gapped_sliver_within_nearest_radius_is_reassigned_not_dropped(self):
|
||||||
|
# A 300 m² sliver 1.2 m from its only neighbour N — beyond the border probe
|
||||||
|
# but inside the nearest-fallback radius. Before the gather buffer was
|
||||||
|
# widened to the accept radius, the old (smaller) gather buffer never
|
||||||
|
# returned N, so the sliver fell through to the drop branch. Now it is
|
||||||
|
# reassigned to N, conserving coverage.
|
||||||
|
x_main = _mbox(0, 0, 100, 100) # far from the sliver
|
||||||
|
x_sliver = _mbox(278.8, 50, 298.8, 65) # 300 m², right edge at x=298.8
|
||||||
|
x = MultiPolygon([x_main, x_sliver])
|
||||||
|
n = _mbox(300, 0, 400, 200) # left edge at x=300 -> 1.2 m gap
|
||||||
|
before = unary_union([x, n]).area
|
||||||
|
out = _as_dict(_eliminate_small_detached_parts([("X", x), ("N", n)]))
|
||||||
|
assert out["X"].geom_type == "Polygon", "sliver should leave X"
|
||||||
|
assert out["N"].contains(x_sliver.representative_point()), "absorbed into N"
|
||||||
|
after = unary_union([out["X"], out["N"]]).area
|
||||||
|
assert after == pytest.approx(before, rel=1e-6), "coverage conserved, not dropped"
|
||||||
|
|
||||||
|
def test_approx_area_matches_real_metric_area(self):
|
||||||
|
# The latitude-scaled area approximation should be within a few percent of
|
||||||
|
# the true projected area for a building-scale polygon.
|
||||||
|
import pyproj
|
||||||
|
from shapely.ops import transform as transform_geometry
|
||||||
|
|
||||||
|
poly = _mbox(0, 0, 50, 60) # ~3000 m²
|
||||||
|
to_bng = pyproj.Transformer.from_crs(
|
||||||
|
"EPSG:4326", "EPSG:27700", always_xy=True
|
||||||
|
)
|
||||||
|
true_m2 = transform_geometry(to_bng.transform, poly).area
|
||||||
|
assert _approx_area_m2(poly) == pytest.approx(true_m2, rel=0.02)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Greenspace subtraction is connectivity-preserving
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestGreenspaceReconnect:
|
||||||
|
"""Greenspace that CROSSES a postcode must not shatter it: a narrow strip is
|
||||||
|
bridged back (postcode stays one piece), a wide barrier is left subtracted
|
||||||
|
(postcode genuinely splits)."""
|
||||||
|
|
||||||
|
def test_narrow_strip_reconnects_postcode(self):
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
postcode = box(0, 0, 200, 100)
|
||||||
|
strip = box(95, 0, 105, 100) # 10 m wide green strip crossing the postcode
|
||||||
|
# Sanity: a plain difference WOULD split it into two parts.
|
||||||
|
assert postcode.difference(strip).geom_type == "MultiPolygon"
|
||||||
|
|
||||||
|
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
|
||||||
|
assert result.geom_type == "Polygon", "narrow strip should be bridged"
|
||||||
|
assert result.is_valid
|
||||||
|
|
||||||
|
def test_wide_barrier_keeps_postcode_split(self):
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
postcode = box(0, 0, 200, 100)
|
||||||
|
barrier = box(60, 0, 130, 100) # 70 m wide — beyond 2x the bridge width
|
||||||
|
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
|
||||||
|
assert result.geom_type == "MultiPolygon", "wide barrier should stay split"
|
||||||
|
assert len(result.geoms) == 2
|
||||||
|
|
||||||
|
def test_edge_greenspace_still_trimmed(self):
|
||||||
|
# Greenspace on the edge (not crossing) is trimmed as before; reconnect is
|
||||||
|
# a no-op because the result stays a single piece.
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
postcode = box(0, 0, 100, 100) # 10000 m²
|
||||||
|
park = box(60, 0, 100, 100) # 4000 m² on the right edge
|
||||||
|
result = subtract_greenspace(postcode, STRtree([park]), [park])
|
||||||
|
assert result.geom_type == "Polygon"
|
||||||
|
assert result.area == pytest.approx(6000, rel=0.01)
|
||||||
|
|
||||||
|
def test_result_is_always_polygonal(self):
|
||||||
|
# Regression: _reconnect_split must never return a GeometryCollection
|
||||||
|
# (line/point debris) — downstream to_wgs84_geojson_multi would truncate a
|
||||||
|
# GC to a single part, silently dropping substantial pieces. Sweep many
|
||||||
|
# strip widths/offsets (incl. coincident-edge-prone integer geometries).
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
postcode = box(0, 0, 200, 120)
|
||||||
|
for w in (2, 5, 8, 10, 25, 49, 50, 51, 60, 90):
|
||||||
|
for x0 in (40, 70, 95, 100, 130):
|
||||||
|
strip = box(x0, -10, x0 + w, 130)
|
||||||
|
result = subtract_greenspace(postcode, STRtree([strip]), [strip])
|
||||||
|
assert result.geom_type in ("Polygon", "MultiPolygon"), (
|
||||||
|
f"w={w} x0={x0} produced {result.geom_type}"
|
||||||
|
)
|
||||||
|
assert result.is_valid and not result.is_empty
|
||||||
|
|
||||||
|
def test_wide_barrier_preserves_all_substantial_parts(self):
|
||||||
|
# The motivating case for the GC fix: a wide barrier genuinely splits the
|
||||||
|
# postcode; ALL substantial parts must survive (not be truncated to one).
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
postcode = box(0, 0, 300, 100)
|
||||||
|
barrier = box(120, 0, 200, 100) # 80 m wide -> beyond 2x bridge
|
||||||
|
result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
|
||||||
|
assert result.geom_type == "MultiPolygon"
|
||||||
|
areas = sorted(p.area for p in result.geoms)
|
||||||
|
assert areas == pytest.approx([10000, 12000], rel=0.01) # both banks kept
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Augment properties.parquet with estimated current prices.
|
"""Estimate current prices for the merged properties, as a standalone artifact.
|
||||||
|
|
||||||
For properties with a known prior sale, applies the repeat-sales price index
|
For properties with a known prior sale, applies the repeat-sales price index
|
||||||
to adjust the last known price to the current date, then blends with kNN
|
to adjust the last known price to the current date, then blends with kNN
|
||||||
|
|
@ -6,8 +6,13 @@ estimates from nearby recently-sold properties. Includes:
|
||||||
- Capping extreme index adjustments
|
- Capping extreme index adjustments
|
||||||
- kNN spatial blending
|
- kNN spatial blending
|
||||||
|
|
||||||
Modifies properties.parquet in-place. Temporarily joins postcode.parquet
|
Reads the slim price_inputs.parquet (built by property_base, independently of
|
||||||
for lat/lon needed by kNN, then drops those columns before writing.
|
merge's area features) plus postcode.parquet for the lat/lon kNN needs, and
|
||||||
|
writes a slim price_estimates.parquet of just the natural key (Postcode +
|
||||||
|
coalesced address) and "Estimated current price" / "Est. price per sqm".
|
||||||
|
join_price_estimates.py joins those two columns back onto properties.parquet.
|
||||||
|
Because the inputs do not depend on merge's area columns, adding such a column
|
||||||
|
does not invalidate this step.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
|
@ -26,12 +31,27 @@ from pipeline.transform.price_estimation.knn import (
|
||||||
from pipeline.transform.price_estimation.utils import (
|
from pipeline.transform.price_estimation.utils import (
|
||||||
CURRENT_FRAC_YEAR,
|
CURRENT_FRAC_YEAR,
|
||||||
CURRENT_YEAR,
|
CURRENT_YEAR,
|
||||||
|
ESTIMATE_COLUMNS,
|
||||||
|
JOIN_KEYS,
|
||||||
MAX_LOG_ADJUSTMENT,
|
MAX_LOG_ADJUSTMENT,
|
||||||
interpolate_log_index,
|
interpolate_log_index,
|
||||||
|
join_address_expr,
|
||||||
sector_expr,
|
sector_expr,
|
||||||
type_group_expr,
|
type_group_expr,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Columns estimate reads from price_inputs.parquet. The two address columns are
|
||||||
|
# only carried to build the natural join key (Postcode + coalesced address).
|
||||||
|
INPUT_COLUMNS = [
|
||||||
|
"Postcode",
|
||||||
|
"Property type",
|
||||||
|
"Total floor area (sqm)",
|
||||||
|
"Last known price",
|
||||||
|
"Date of last transaction",
|
||||||
|
"Address per Property Register",
|
||||||
|
"Address per EPC",
|
||||||
|
]
|
||||||
|
|
||||||
MAX_KNN_TO_INDEX_RATIO = 2.0
|
MAX_KNN_TO_INDEX_RATIO = 2.0
|
||||||
MIN_KNN_TO_INDEX_RATIO = 0.5
|
MIN_KNN_TO_INDEX_RATIO = 0.5
|
||||||
# Cap the final estimate at this multiple of the last known price as a guard
|
# Cap the final estimate at this multiple of the last known price as a guard
|
||||||
|
|
@ -161,13 +181,14 @@ def guarded_blend_estimates(
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Augment properties.parquet with estimated current prices"
|
description="Estimate current prices for the merged properties"
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--properties",
|
"--input",
|
||||||
type=Path,
|
type=Path,
|
||||||
required=True,
|
required=True,
|
||||||
help="Path to properties.parquet (modified in-place)",
|
help="Path to price_inputs.parquet (slim per-dwelling inputs from "
|
||||||
|
"property_base)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--postcodes",
|
"--postcodes",
|
||||||
|
|
@ -178,22 +199,23 @@ def main():
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--index", type=Path, required=True, help="Path to price_index.parquet"
|
"--index", type=Path, required=True, help="Path to price_index.parquet"
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Output price_estimates.parquet (natural key + estimate columns)",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
print("Loading properties.parquet...")
|
print("Loading price inputs (projection)...")
|
||||||
df = pl.read_parquet(args.properties)
|
df = pl.read_parquet(args.input, columns=INPUT_COLUMNS)
|
||||||
print(f" {len(df):,} rows, {len(df.columns)} columns")
|
print(f" {len(df):,} rows, {len(INPUT_COLUMNS)} input columns")
|
||||||
|
|
||||||
# Join lat/lon from postcode.parquet for kNN spatial queries
|
# Join lat/lon from postcode.parquet for kNN spatial queries
|
||||||
postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon")
|
postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon")
|
||||||
df = df.join(postcodes, on="Postcode", how="left")
|
df = df.join(postcodes, on="Postcode", how="left")
|
||||||
print(f" Joined lat/lon from {len(postcodes):,} postcodes")
|
print(f" Joined lat/lon from {len(postcodes):,} postcodes")
|
||||||
|
|
||||||
# Drop existing estimated columns if re-running
|
|
||||||
for col in ["Estimated current price", "Est. price per sqm"]:
|
|
||||||
if col in df.columns:
|
|
||||||
df = df.drop(col)
|
|
||||||
|
|
||||||
# Derive helper columns
|
# Derive helper columns
|
||||||
df = df.with_columns(
|
df = df.with_columns(
|
||||||
sector_expr().alias("_sector"),
|
sector_expr().alias("_sector"),
|
||||||
|
|
@ -355,16 +377,15 @@ def main():
|
||||||
.alias("Est. price per sqm"),
|
.alias("Est. price per sqm"),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Drop all temporary columns and joined lat/lon (those belong in postcode.parquet)
|
# Emit only the natural join key and the two estimate columns.
|
||||||
temp_cols = [c for c in df.columns if c.startswith("_") or c.startswith("log_idx_")]
|
# join_price_estimates.py joins these back onto properties.parquet.
|
||||||
df = df.drop(temp_cols).drop("lat", "lon")
|
result = df.with_columns(join_address_expr()).select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
|
||||||
|
|
||||||
df.write_parquet(args.properties)
|
result.write_parquet(args.output)
|
||||||
size_mb = args.properties.stat().st_size / (1024 * 1024)
|
size_mb = args.output.stat().st_size / (1024 * 1024)
|
||||||
print(f"\nWrote {args.properties} ({size_mb:.1f} MB)")
|
print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
|
||||||
print(
|
n_priced = result.filter(pl.col("Estimated current price").is_not_null()).height
|
||||||
f" {len(df):,} rows, {len(df.columns)} columns (including 'Estimated current price')"
|
print(f" {len(result):,} rows, {n_priced:,} with an estimated current price")
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,26 @@ LATEST_COMPLETE_YEAR = CURRENT_YEAR - 1
|
||||||
_today = date.today()
|
_today = date.today()
|
||||||
CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12
|
CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12
|
||||||
|
|
||||||
|
# The two columns price estimation contributes to properties.parquet, kept here
|
||||||
|
# so both the producer (estimate) and the joiner (join_price_estimates) agree.
|
||||||
|
ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
|
||||||
|
|
||||||
|
# Natural join key from estimates back onto properties: postcode plus the
|
||||||
|
# coalesced register/EPC address. This is unique and non-null on the deduped
|
||||||
|
# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
|
||||||
|
# estimates 1:1 onto properties regardless of row order — estimates are computed
|
||||||
|
# from a separate price_inputs.parquet, so a positional key would not line up.
|
||||||
|
JOIN_ADDRESS = "_join_address"
|
||||||
|
JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
|
||||||
|
|
||||||
|
|
||||||
|
def join_address_expr() -> pl.Expr:
|
||||||
|
"""The coalesced address half of the natural key, aliased to JOIN_ADDRESS."""
|
||||||
|
return pl.coalesce("Address per Property Register", "Address per EPC").alias(
|
||||||
|
JOIN_ADDRESS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Cap on log(index_ratio) to prevent wild estimates from thin sectors
|
# Cap on log(index_ratio) to prevent wild estimates from thin sectors
|
||||||
MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change
|
MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change
|
||||||
TERRACE_TYPES = [
|
TERRACE_TYPES = [
|
||||||
|
|
|
||||||
217
pipeline/transform/property_base.py
Normal file
217
pipeline/transform/property_base.py
Normal file
|
|
@ -0,0 +1,217 @@
|
||||||
|
"""Shared property base: the dwelling universe before any area enrichment.
|
||||||
|
|
||||||
|
This is the single source of truth for *which* dwellings exist and their
|
||||||
|
intrinsic, source-level attributes (price, floor area, type, addresses). Both
|
||||||
|
``merge`` (which enriches it with postcode/LSOA-keyed area features to build
|
||||||
|
properties.parquet) and price estimation (which only needs the intrinsic
|
||||||
|
columns) start from exactly these rows, so estimates computed here line up 1:1
|
||||||
|
with the final properties by the natural key ``(Postcode, coalesced address)``.
|
||||||
|
|
||||||
|
Living in its own module is what lets price estimation be *cached* across
|
||||||
|
merge changes: ``price_inputs.parquet`` depends only on epc_pp + arcgis + this
|
||||||
|
file, so adding an area column to merge.py does not invalidate it and the
|
||||||
|
expensive index/kNN steps are skipped.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
from pipeline.utils.postcode_mapping import build_postcode_mapping
|
||||||
|
|
||||||
|
MIN_FLOOR_AREA_M2 = 10
|
||||||
|
|
||||||
|
# Columns price estimation reads, with the final (properties.parquet) names so
|
||||||
|
# index.py/estimate.py and the join all speak the same schema. The two address
|
||||||
|
# columns form the natural join key (Postcode + their coalesce).
|
||||||
|
PRICE_INPUT_SELECT = [
|
||||||
|
pl.col("postcode").alias("Postcode"),
|
||||||
|
pl.col("total_floor_area").alias("Total floor area (sqm)"),
|
||||||
|
pl.col("latest_price").alias("Last known price"),
|
||||||
|
pl.col("date_of_transfer").alias("Date of last transaction"),
|
||||||
|
"historical_prices",
|
||||||
|
pl.col("pp_address").alias("Address per Property Register"),
|
||||||
|
pl.col("epc_address").alias("Address per EPC"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
|
||||||
|
"""Return the supported postcode universe with geography join keys."""
|
||||||
|
return (
|
||||||
|
arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
|
||||||
|
.filter(pl.col("doterm").is_null())
|
||||||
|
.select(
|
||||||
|
pl.col("pcds").alias("postcode"),
|
||||||
|
"lat",
|
||||||
|
pl.col("long").alias("lon"),
|
||||||
|
"ctry25cd",
|
||||||
|
pl.col("lsoa21cd").alias("lsoa21"),
|
||||||
|
pl.col("oa21cd").alias("oa21"),
|
||||||
|
pl.col("pcon24cd").alias("pcon"),
|
||||||
|
)
|
||||||
|
.drop_nulls(["postcode"])
|
||||||
|
.unique(["postcode"])
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _remap_terminated_postcodes(
|
||||||
|
wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
|
||||||
|
) -> pl.LazyFrame:
|
||||||
|
return (
|
||||||
|
wide.join(
|
||||||
|
postcode_mapping,
|
||||||
|
left_on="postcode",
|
||||||
|
right_on="old_postcode",
|
||||||
|
how="left",
|
||||||
|
)
|
||||||
|
.with_columns(
|
||||||
|
pl.coalesce("new_postcode", "postcode").alias("postcode"),
|
||||||
|
)
|
||||||
|
.drop("new_postcode")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||||
|
"""Keep one row per (postcode, address) — the most-recent transaction.
|
||||||
|
|
||||||
|
The terminated-postcode remap can map two distinct postcodes onto one active
|
||||||
|
successor, collapsing the same physical address onto a single
|
||||||
|
(postcode, address) key with conflicting sale records. Keep the row with the
|
||||||
|
latest date_of_transfer so the headline price/date reflect the most recent
|
||||||
|
transaction; genuinely distinct addresses are untouched.
|
||||||
|
|
||||||
|
The dedup key coalesces the price-paid address with the EPC address: EPC-only
|
||||||
|
dwellings (never sold) have a null pp_address, so keying on pp_address alone
|
||||||
|
would collapse EVERY EPC-only dwelling in a postcode onto one
|
||||||
|
(postcode, null) key and silently drop all but one. Each dwelling's coalesced
|
||||||
|
address is unique within its postcode (the EPC frame is deduped on
|
||||||
|
address+postcode upstream), so the coalesced key keeps them distinct while
|
||||||
|
leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
|
||||||
|
a sale exists.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
wide.with_columns(
|
||||||
|
pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
|
||||||
|
)
|
||||||
|
.sort("date_of_transfer", descending=True, nulls_last=True)
|
||||||
|
.unique(
|
||||||
|
subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
|
||||||
|
)
|
||||||
|
.drop("_dedupe_address")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_to_active_english_postcodes(
|
||||||
|
wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
|
||||||
|
) -> pl.LazyFrame:
|
||||||
|
return wide.join(active_postcodes, on="postcode", how="semi")
|
||||||
|
|
||||||
|
|
||||||
|
def property_type_expr() -> pl.Expr:
|
||||||
|
"""Unaliased property-type expression: prefer EPC, fall back to price-paid.
|
||||||
|
|
||||||
|
For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer
|
||||||
|
detail. Depends only on intrinsic base columns (epc_property_type,
|
||||||
|
pp_property_type, built_form), so merge and price_inputs derive the same
|
||||||
|
value. Callers alias it ("property_type" in merge, "Property type" in
|
||||||
|
price_inputs).
|
||||||
|
"""
|
||||||
|
bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
|
||||||
|
["NO DATA!", "Not Recorded"]
|
||||||
|
)
|
||||||
|
has_epc = pl.col("epc_property_type").is_not_null()
|
||||||
|
is_house = pl.col("epc_property_type") == "House"
|
||||||
|
return (
|
||||||
|
pl.when(has_epc & is_house & ~bad_built_form)
|
||||||
|
.then(pl.col("built_form"))
|
||||||
|
.when(has_epc & is_house)
|
||||||
|
.then(pl.col("pp_property_type"))
|
||||||
|
.when(has_epc)
|
||||||
|
.then(pl.col("epc_property_type"))
|
||||||
|
.otherwise(pl.col("pp_property_type"))
|
||||||
|
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
|
||||||
|
# collapse terrace sub-types, and fold rare types into "Other"
|
||||||
|
.replace(
|
||||||
|
{
|
||||||
|
"Flat": "Flats/Maisonettes",
|
||||||
|
"Maisonette": "Flats/Maisonettes",
|
||||||
|
"End-Terrace": "Terraced",
|
||||||
|
"Mid-Terrace": "Terraced",
|
||||||
|
"Enclosed End-Terrace": "Terraced",
|
||||||
|
"Enclosed Mid-Terrace": "Terraced",
|
||||||
|
"Bungalow": "Other",
|
||||||
|
"Park home": "Other",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_postcode_centroids(arcgis_path: Path) -> pl.LazyFrame:
|
||||||
|
"""One row per active-English postcode with its lat/lon, from arcgis.
|
||||||
|
|
||||||
|
This is the lat/lon source price estimation needs (index sector centroids,
|
||||||
|
kNN). It is the same per-postcode lat/lon merge writes into postcode.parquet
|
||||||
|
(both come from arcgis), but built straight from arcgis so the index/estimate
|
||||||
|
steps do not depend on the merge output — adding an area column to merge
|
||||||
|
therefore does not invalidate the expensive price index/kNN.
|
||||||
|
"""
|
||||||
|
return _active_english_postcode_area(pl.scan_parquet(arcgis_path)).select(
|
||||||
|
pl.col("postcode").alias("Postcode"), "lat", "lon"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_property_base(epc_pp_path: Path, arcgis_path: Path) -> pl.LazyFrame:
|
||||||
|
"""The deduped, active-English dwelling universe from epc_pp + arcgis.
|
||||||
|
|
||||||
|
Floor filter -> terminated-postcode remap -> collapse-dedupe -> restrict to
|
||||||
|
the active English postcode universe. Returns a LazyFrame with the original
|
||||||
|
epc_pp column names; merge enriches it, the CLI projects it to price_inputs.
|
||||||
|
"""
|
||||||
|
wide = pl.scan_parquet(epc_pp_path).filter(
|
||||||
|
pl.col("total_floor_area").is_null()
|
||||||
|
| (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
|
||||||
|
)
|
||||||
|
postcode_mapping = build_postcode_mapping(arcgis_path)
|
||||||
|
wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
|
||||||
|
wide = _dedupe_collapsed_properties(wide)
|
||||||
|
active_postcodes = (
|
||||||
|
_active_english_postcode_area(pl.scan_parquet(arcgis_path))
|
||||||
|
.select("postcode")
|
||||||
|
.unique()
|
||||||
|
)
|
||||||
|
return _filter_to_active_english_postcodes(wide, active_postcodes)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Write the slim price-estimation inputs from epc_pp + arcgis"
|
||||||
|
)
|
||||||
|
parser.add_argument("--epc-pp", type=Path, required=True)
|
||||||
|
parser.add_argument("--arcgis", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output", type=Path, required=True, help="price_inputs.parquet output"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--centroids",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="postcode_centroids.parquet output (Postcode, lat, lon)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
base = build_property_base(args.epc_pp, args.arcgis)
|
||||||
|
price_inputs = base.with_columns(
|
||||||
|
property_type_expr().alias("Property type")
|
||||||
|
).select(*PRICE_INPUT_SELECT, "Property type")
|
||||||
|
price_inputs.sink_parquet(args.output)
|
||||||
|
n = pl.scan_parquet(args.output).select(pl.len()).collect().item()
|
||||||
|
print(f"Wrote {args.output} ({args.output.stat().st_size / 1e6:.1f} MB), {n:,} dwellings")
|
||||||
|
|
||||||
|
build_postcode_centroids(args.arcgis).sink_parquet(args.centroids)
|
||||||
|
n_pc = pl.scan_parquet(args.centroids).select(pl.len()).collect().item()
|
||||||
|
print(f"Wrote {args.centroids} ({args.centroids.stat().st_size / 1e6:.1f} MB), {n_pc:,} postcodes")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
81
pipeline/transform/test_area_crime_averages.py
Normal file
81
pipeline/transform/test_area_crime_averages.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
from pipeline.transform.area_crime_averages import (
|
||||||
|
NATIONAL_AREA,
|
||||||
|
SCOPE_NATIONAL,
|
||||||
|
SCOPE_OUTCODE,
|
||||||
|
SCOPE_SECTOR,
|
||||||
|
compute_area_crime_averages,
|
||||||
|
)
|
||||||
|
|
||||||
|
_BURGLARY = "Burglary (/yr, 7y)"
|
||||||
|
_ROBBERY = "Robbery (/yr, 7y)"
|
||||||
|
|
||||||
|
|
||||||
|
def _postcodes() -> pl.LazyFrame:
|
||||||
|
return pl.LazyFrame(
|
||||||
|
{
|
||||||
|
"Postcode": ["E14 2DG", "E14 2AB", "E14 9XY", "M1 1AE", "E14 2ZZ"],
|
||||||
|
# E14 9XY has no usable crime data; E14 2AB lacks robbery; E14 2ZZ has
|
||||||
|
# crime but (below) no properties, so it must not weight any average.
|
||||||
|
_BURGLARY: [10.0, 20.0, None, 5.0, 100.0],
|
||||||
|
_ROBBERY: [2.0, None, None, 1.0, 50.0],
|
||||||
|
# An unrelated column proves only the crime columns are averaged.
|
||||||
|
"Median age": [40.0, 41.0, 42.0, 30.0, 99.0],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _properties() -> pl.LazyFrame:
|
||||||
|
# Property rows per postcode become the weights (3 / 1 / 2 / 4). E14 2ZZ has
|
||||||
|
# none, so it is excluded entirely.
|
||||||
|
postcodes = ["E14 2DG"] * 3 + ["E14 2AB"] + ["E14 9XY"] * 2 + ["M1 1AE"] * 4
|
||||||
|
return pl.LazyFrame({"Postcode": postcodes})
|
||||||
|
|
||||||
|
|
||||||
|
def _row(df: pl.DataFrame, scope: str, area: str) -> dict:
|
||||||
|
matched = df.filter((pl.col("scope") == scope) & (pl.col("area") == area))
|
||||||
|
assert matched.height == 1, f"expected one {scope} row for {area!r}"
|
||||||
|
return matched.to_dicts()[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_property_weighted_means_skip_nulls():
|
||||||
|
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||||
|
|
||||||
|
national = _row(result, SCOPE_NATIONAL, NATIONAL_AREA)
|
||||||
|
# Burglary: (10*3 + 20*1 + 5*4) / (3+1+4) = 70/8; E14 9XY null dilutes nothing.
|
||||||
|
assert national[_BURGLARY] == 8.75
|
||||||
|
# Robbery: (2*3 + 1*4) / (3+4) = 10/7; both null postcodes are excluded from
|
||||||
|
# the numerator AND the denominator.
|
||||||
|
assert national[_ROBBERY] == pl.Series([10.0 / 7.0]).cast(pl.Float32).item()
|
||||||
|
|
||||||
|
outcode = _row(result, SCOPE_OUTCODE, "E14")
|
||||||
|
assert outcode[_BURGLARY] == 12.5 # (10*3 + 20*1) / 4
|
||||||
|
assert outcode[_ROBBERY] == 2.0 # only E14 2DG has robbery (2 * 3 / 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sector_aggregation_and_all_null_rows_dropped():
|
||||||
|
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||||
|
|
||||||
|
sector = _row(result, SCOPE_SECTOR, "E14 2")
|
||||||
|
assert sector[_BURGLARY] == 12.5
|
||||||
|
assert sector[_ROBBERY] == 2.0
|
||||||
|
|
||||||
|
# E14 9XY has properties but no crime data at all, so its sector "E14 9" is
|
||||||
|
# all-null and must be dropped rather than reported as a known area.
|
||||||
|
assert result.filter(pl.col("area") == "E14 9").height == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_postcodes_without_properties_are_excluded():
|
||||||
|
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||||
|
|
||||||
|
# E14 2ZZ carries crime values but no properties; including it would pull the
|
||||||
|
# E14 outcode burglary mean toward its 100.0. It must contribute nothing.
|
||||||
|
outcode = _row(result, SCOPE_OUTCODE, "E14")
|
||||||
|
assert outcode[_BURGLARY] == 12.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_crime_columns_are_emitted():
|
||||||
|
result = compute_area_crime_averages(_postcodes(), _properties())
|
||||||
|
assert set(result.columns) == {"scope", "area", _BURGLARY, _ROBBERY}
|
||||||
|
assert result.schema[_BURGLARY] == pl.Float32
|
||||||
|
|
@ -1,13 +1,10 @@
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import polars as pl
|
import polars as pl
|
||||||
import pytest
|
import pytest
|
||||||
import shapely
|
|
||||||
from pyproj import Transformer
|
from pyproj import Transformer
|
||||||
|
|
||||||
from pipeline.transform.crime_spatial import transform_crime_spatial
|
from pipeline.transform.crime_spatial import transform_crime_spatial
|
||||||
from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
|
|
||||||
|
|
||||||
_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
|
_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
|
||||||
|
|
||||||
|
|
@ -16,6 +13,10 @@ _CSV_HEADER = (
|
||||||
"LSOA code,LSOA name,Crime type,Last outcome category,Context"
|
"LSOA code,LSOA name,Crime type,Last outcome category,Context"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Average-annual-count crime column name for a window (the filterable feature).
|
||||||
|
def _raw(t: str, window: str = "7y") -> str:
|
||||||
|
return f"{t} (/yr, {window})"
|
||||||
|
|
||||||
|
|
||||||
def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]:
|
def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]:
|
||||||
lon, lat = _TO_WGS84.transform(x, y)
|
lon, lat = _TO_WGS84.transform(x, y)
|
||||||
|
|
@ -39,12 +40,12 @@ def _write_boundaries(units_dir, features_by_district: dict[str, list[dict]]) ->
|
||||||
(units_dir / f"{district}.geojson").write_text(json.dumps(collection))
|
(units_dir / f"{district}.geojson").write_text(json.dumps(collection))
|
||||||
|
|
||||||
|
|
||||||
def _crime_row(month: str, x, y, crime_type: str) -> str:
|
def _crime_row(month: str, x, y, crime_type: str, location="On or near X", outcome="U") -> str:
|
||||||
if x is None or y is None:
|
if x is None or y is None:
|
||||||
lon, lat = "", ""
|
lon, lat = "", ""
|
||||||
else:
|
else:
|
||||||
lon, lat = _bng_to_wgs84(x, y)
|
lon, lat = _bng_to_wgs84(x, y)
|
||||||
return f",{month},F,F,{lon},{lat},On or near X,E01000001,L,{crime_type},U,"
|
return f",{month},F,F,{lon},{lat},{location},E01000001,L,{crime_type},{outcome},"
|
||||||
|
|
||||||
|
|
||||||
def _write_month(
|
def _write_month(
|
||||||
|
|
@ -59,10 +60,22 @@ def _write_month(
|
||||||
|
|
||||||
|
|
||||||
def _run(tmp_path, crime, units, **kwargs):
|
def _run(tmp_path, crime, units, **kwargs):
|
||||||
output = tmp_path / "crime_by_postcode.parquet"
|
"""Run the transform and return (crime, by_year, records) DataFrames.
|
||||||
|
|
||||||
|
The crime table carries the average-annual-count columns ("{type} (/yr, …)"),
|
||||||
|
i.e. the raw, absolute number of recorded incidents per year.
|
||||||
|
"""
|
||||||
|
crime_out = tmp_path / "crime_by_postcode.parquet"
|
||||||
by_year = tmp_path / "crime_by_postcode_by_year.parquet"
|
by_year = tmp_path / "crime_by_postcode_by_year.parquet"
|
||||||
transform_crime_spatial(crime, units, output, by_year, buffer_m=50.0, **kwargs)
|
records = tmp_path / "crime_records.parquet"
|
||||||
return pl.read_parquet(output), pl.read_parquet(by_year)
|
transform_crime_spatial(
|
||||||
|
crime, units, crime_out, by_year, records, buffer_m=50.0, **kwargs
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
pl.read_parquet(crime_out),
|
||||||
|
pl.read_parquet(by_year),
|
||||||
|
pl.read_parquet(records),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_buffer_overlap_counts_for_each_postcode(tmp_path):
|
def test_buffer_overlap_counts_for_each_postcode(tmp_path):
|
||||||
|
|
@ -95,17 +108,74 @@ def test_buffer_overlap_counts_for_each_postcode(tmp_path):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
avg_df, _ = _run(tmp_path, crime, units)
|
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||||
# Single covered month -> pooled rate x12.
|
# Single covered month -> pooled raw rate x12.
|
||||||
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == 12.0
|
assert rows["AB1 1AA"][_raw("Burglary")] == 12.0
|
||||||
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == 12.0
|
assert rows["AB1 1AB"][_raw("Burglary")] == 12.0
|
||||||
assert rows["AB1 1AA"]["Robbery (avg/yr)"] == 0.0
|
assert rows["AB1 1AA"][_raw("Robbery")] == 0.0
|
||||||
# Only the 49m robbery counts for C; the 51m one and the blank row do not.
|
# Only the 49m robbery counts for C; the 51m one and the blank row do not.
|
||||||
assert rows["AB1 1AC"]["Robbery (avg/yr)"] == 12.0
|
assert rows["AB1 1AC"][_raw("Robbery")] == 12.0
|
||||||
assert rows["AB1 1AC"]["Burglary (avg/yr)"] == 0.0
|
assert rows["AB1 1AC"][_raw("Burglary")] == 0.0
|
||||||
# Anti-social behaviour had no coordinate -> nobody gets it.
|
# Anti-social behaviour had no coordinate -> nobody gets it.
|
||||||
assert all(r["Anti-social behaviour (avg/yr)"] == 0.0 for r in rows.values())
|
assert all(r[_raw("Anti-social behaviour")] == 0.0 for r in rows.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_counts_are_not_area_normalised(tmp_path):
|
||||||
|
# Three postcodes of very different footprint, each with exactly one incident
|
||||||
|
# in its buffer. The raw count must be 12/yr for ALL of them: area
|
||||||
|
# normalisation has been removed, so footprint no longer changes the number.
|
||||||
|
units = tmp_path / "units"
|
||||||
|
_write_boundaries(
|
||||||
|
units,
|
||||||
|
{
|
||||||
|
"AB1": [
|
||||||
|
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
|
||||||
|
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20
|
||||||
|
_square_feature("AB1 1AC", 5000, 5000, 5040, 5040), # 40x40
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
crime = tmp_path / "crime"
|
||||||
|
_write_month(
|
||||||
|
crime,
|
||||||
|
"2024-01",
|
||||||
|
[
|
||||||
|
_crime_row("2024-01", 1005, 1005, "Burglary"),
|
||||||
|
_crime_row("2024-01", 3005, 3010, "Burglary"),
|
||||||
|
_crime_row("2024-01", 5020, 5020, "Burglary"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
raw_df, _, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
|
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||||
|
for pc in ("AB1 1AA", "AB1 1AB", "AB1 1AC"):
|
||||||
|
assert rows[pc][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||||
|
|
||||||
|
|
||||||
|
def test_windows_pool_only_recent_years(tmp_path):
|
||||||
|
# 2-year window vs 7-year window. An incident in the latest year sits in both
|
||||||
|
# windows; one 6 years back sits only in the 7-year window.
|
||||||
|
units = tmp_path / "units"
|
||||||
|
_write_boundaries(
|
||||||
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
|
)
|
||||||
|
crime = tmp_path / "crime"
|
||||||
|
# 12 covered months in 2019 (1 burglary), 12 in 2025 (1 burglary). Latest =
|
||||||
|
# 2025: 7y window = 2019..2025 (both), 2y window = 2024..2025 (only 2025).
|
||||||
|
for month in range(1, 13):
|
||||||
|
ym19 = f"2019-{month:02d}"
|
||||||
|
ym25 = f"2025-{month:02d}"
|
||||||
|
_write_month(crime, ym19, [_crime_row(ym19, 1005, 1005, "Burglary")] if month == 1 else [])
|
||||||
|
_write_month(crime, ym25, [_crime_row(ym25, 1005, 1005, "Burglary")] if month == 1 else [])
|
||||||
|
|
||||||
|
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||||
|
row = raw_df.row(0, named=True)
|
||||||
|
# 7y: 2 incidents over 24 covered months -> 1/yr.
|
||||||
|
assert row[_raw("Burglary", "7y")] == pytest.approx(1.0, abs=0.05)
|
||||||
|
# 2y: 1 incident over 12 covered months -> 1/yr (the 2019 one is excluded).
|
||||||
|
assert row[_raw("Burglary", "2y")] == pytest.approx(1.0, abs=0.05)
|
||||||
|
|
||||||
|
|
||||||
def test_by_year_annualises_and_rolls_up(tmp_path):
|
def test_by_year_annualises_and_rolls_up(tmp_path):
|
||||||
|
|
@ -115,7 +185,6 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
||||||
)
|
)
|
||||||
|
|
||||||
crime = tmp_path / "crime"
|
crime = tmp_path / "crime"
|
||||||
# Point at the centre of AB1 1AA, well inside its buffer.
|
|
||||||
_write_month(
|
_write_month(
|
||||||
crime,
|
crime,
|
||||||
"2023-01",
|
"2023-01",
|
||||||
|
|
@ -134,7 +203,7 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
_, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
_, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
assert by_year_df.height == 1
|
assert by_year_df.height == 1
|
||||||
cols = set(by_year_df.columns)
|
cols = set(by_year_df.columns)
|
||||||
assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols
|
assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols
|
||||||
|
|
@ -150,77 +219,14 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
|
||||||
# 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12).
|
# 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12).
|
||||||
assert serious[2023] == 24.0
|
assert serious[2023] == 24.0
|
||||||
assert serious[2024] == 12.0
|
assert serious[2024] == 12.0
|
||||||
# Coverage calendar: both years published, with their month counts.
|
|
||||||
coverage = {c["year"]: c["months"] for c in row["covered_years"]}
|
coverage = {c["year"]: c["months"] for c in row["covered_years"]}
|
||||||
assert coverage == {2023: 1, 2024: 2}
|
assert coverage == {2023: 1, 2024: 2}
|
||||||
|
|
||||||
|
|
||||||
def test_area_normalisation_divides_out_buffered_catchment(tmp_path):
|
def test_raw_is_pooled_rate_over_covered_months(tmp_path):
|
||||||
# Three postcodes of increasing footprint, each with exactly one incident in
|
# Uneven month coverage: 2023 has 1 month (2 incidents), 2024 has 2 months
|
||||||
# its buffer. Normalisation rescales by median_catchment / buffered_area, so
|
# (2 incidents). The raw figure is the POOLED annualised rate over all covered
|
||||||
# the smallest scores highest and the median-sized one is unchanged -- i.e.
|
# months: 4 incidents / 3 months * 12 = 16/yr.
|
||||||
# the metric is a density. Dividing by the *buffered* catchment (not the raw
|
|
||||||
# polygon) means the fixed buffer-ring floor keeps the spread gentle, so the
|
|
||||||
# tiniest postcode is not blown up out of proportion.
|
|
||||||
units = tmp_path / "units"
|
|
||||||
_write_boundaries(
|
|
||||||
units,
|
|
||||||
{
|
|
||||||
"AB1": [
|
|
||||||
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
|
|
||||||
_square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20 (median)
|
|
||||||
_square_feature("AB1 1AC", 5000, 5000, 5020, 5020), # 20x20
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
crime = tmp_path / "crime"
|
|
||||||
_write_month(
|
|
||||||
crime,
|
|
||||||
"2024-01",
|
|
||||||
[
|
|
||||||
_crime_row("2024-01", 1005, 1005, "Burglary"),
|
|
||||||
_crime_row("2024-01", 3005, 3010, "Burglary"),
|
|
||||||
_crime_row("2024-01", 5010, 5010, "Burglary"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
|
||||||
|
|
||||||
# Re-derive the expected values from the same buffered catchment areas: each
|
|
||||||
# postcode is 12/yr before normalisation, then x (median_buf / buffered_area).
|
|
||||||
postcodes, polygons = load_postcode_polygons(units)
|
|
||||||
buf_area = {
|
|
||||||
pc: float(shapely.area(shapely.buffer(poly, 50.0, quad_segs=8)))
|
|
||||||
for pc, poly in zip(postcodes, polygons)
|
|
||||||
}
|
|
||||||
median_buf = float(np.median(list(buf_area.values())))
|
|
||||||
expected = {pc: 12.0 * median_buf / buf_area[pc] for pc in buf_area}
|
|
||||||
|
|
||||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
|
||||||
for pc, exp in expected.items():
|
|
||||||
assert rows[pc]["Burglary (avg/yr)"] == pytest.approx(exp, abs=0.1)
|
|
||||||
|
|
||||||
# Median catchment unchanged; ordering is by inverse buffered area, but the
|
|
||||||
# buffer-ring floor keeps the spread far below the ~4x raw-area ratio.
|
|
||||||
assert rows["AB1 1AB"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
|
||||||
small = rows["AB1 1AA"]["Burglary (avg/yr)"]
|
|
||||||
big = rows["AB1 1AC"]["Burglary (avg/yr)"]
|
|
||||||
assert small > 12.0 > big
|
|
||||||
assert small / big < 1.5
|
|
||||||
|
|
||||||
# by-year series carries the same normalisation.
|
|
||||||
small_row = by_year_df.filter(pl.col("postcode") == "AB1 1AA").row(0, named=True)
|
|
||||||
assert small_row["Burglary (by year)"] == [
|
|
||||||
{"year": 2024, "count": pytest.approx(expected["AB1 1AA"], abs=0.1)}
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
|
||||||
# Uneven month coverage across years: 2023 has 1 month (2 incidents),
|
|
||||||
# 2024 has 2 months (2 incidents). The headline is the POOLED annualised
|
|
||||||
# rate over all covered months: 4 incidents / 3 months * 12 = 16/yr -- not
|
|
||||||
# the old mean-of-bars (24+12)/2 = 18, which over-weighted thin years.
|
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
|
|
@ -238,10 +244,9 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
||||||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
||||||
_write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")])
|
_write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")])
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
|
|
||||||
avg = avg_df.row(0, named=True)
|
assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(16.0, abs=0.05)
|
||||||
assert avg["Burglary (avg/yr)"] == pytest.approx(16.0, abs=0.05)
|
|
||||||
|
|
||||||
# Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6).
|
# Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6).
|
||||||
row = by_year_df.row(0, named=True)
|
row = by_year_df.row(0, named=True)
|
||||||
|
|
@ -251,8 +256,7 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
|
||||||
|
|
||||||
def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
||||||
# A single robbery in a 24-covered-month window must read as ~0.5/yr (the
|
# A single robbery in a 24-covered-month window must read as ~0.5/yr (the
|
||||||
# long-run pooled rate), NOT 12/yr (the old years-with-incidents mean that
|
# long-run pooled rate), NOT 12/yr.
|
||||||
# inflated sporadic categories by up to ~15x).
|
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
|
|
@ -266,14 +270,10 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
||||||
rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")]
|
rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")]
|
||||||
_write_month(crime, f"{year}-{month:02d}", rows)
|
_write_month(crime, f"{year}-{month:02d}", rows)
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||||
|
|
||||||
avg = avg_df.row(0, named=True)
|
|
||||||
# 1 incident over 24 covered months -> 0.5/yr.
|
# 1 incident over 24 covered months -> 0.5/yr.
|
||||||
assert avg["Robbery (avg/yr)"] == pytest.approx(0.5, abs=0.05)
|
assert raw_df.row(0, named=True)[_raw("Robbery")] == pytest.approx(0.5, abs=0.05)
|
||||||
# The by-year bar still shows the 2023 incident annualised over 12 covered
|
|
||||||
# months (1/yr); 2024 is covered with zero robberies -> no bar, but the
|
|
||||||
# year IS in the coverage list so consumers may render it as a true zero.
|
|
||||||
row = by_year_df.row(0, named=True)
|
row = by_year_df.row(0, named=True)
|
||||||
bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]}
|
bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]}
|
||||||
assert bars == {2023: pytest.approx(1.0, abs=0.05)}
|
assert bars == {2023: pytest.approx(1.0, abs=0.05)}
|
||||||
|
|
@ -283,9 +283,8 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
|
||||||
|
|
||||||
def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
||||||
# Two postcodes policed by different forces. force-a publishes 2023+2024;
|
# Two postcodes policed by different forces. force-a publishes 2023+2024;
|
||||||
# force-b publishes only 2023 (a 2024 gap, like Greater Manchester). The
|
# force-b publishes only 2023 (a 2024 gap). The b-postcode's raw figure must
|
||||||
# b-postcode's headline must pool over force-b's 12 covered months only,
|
# pool over force-b's 12 covered months only.
|
||||||
# and its by-year series must NOT contain a 2024 bar or coverage entry.
|
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
units,
|
units,
|
||||||
|
|
@ -299,25 +298,21 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
||||||
for month in range(1, 13):
|
for month in range(1, 13):
|
||||||
ym23 = f"2023-{month:02d}"
|
ym23 = f"2023-{month:02d}"
|
||||||
ym24 = f"2024-{month:02d}"
|
ym24 = f"2024-{month:02d}"
|
||||||
# force-a covers AB1 in both years; one burglary per month in 2024.
|
|
||||||
_write_month(crime, ym23, [], force="force-a")
|
_write_month(crime, ym23, [], force="force-a")
|
||||||
_write_month(
|
_write_month(
|
||||||
crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a"
|
crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a"
|
||||||
)
|
)
|
||||||
# force-b covers CD1 in 2023 only: one burglary per month.
|
|
||||||
_write_month(
|
_write_month(
|
||||||
crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b"
|
crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b"
|
||||||
)
|
)
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||||
rows = {r["postcode"]: r for r in avg_df.to_dicts()}
|
rows = {r["postcode"]: r for r in raw_df.to_dicts()}
|
||||||
|
|
||||||
# force-a postcode: 12 burglaries over 24 covered months -> 6/yr.
|
# force-a postcode: 12 burglaries over 24 covered months -> 6/yr.
|
||||||
assert rows["AB1 1AA"]["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
assert rows["AB1 1AA"][_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
|
||||||
# force-b postcode: 12 burglaries over 12 covered months -> 12/yr. Under
|
# force-b postcode: 12 burglaries over 12 covered months -> 12/yr.
|
||||||
# the old global calendar this would have been diluted to 6/yr by the
|
assert rows["CD1 1AA"][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||||
# uncovered 2024.
|
|
||||||
assert rows["CD1 1AA"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
|
||||||
|
|
||||||
by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()}
|
by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()}
|
||||||
b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]}
|
b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]}
|
||||||
|
|
@ -328,59 +323,10 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
|
||||||
assert a_coverage == {2023: 12, 2024: 12}
|
assert a_coverage == {2023: 12, 2024: 12}
|
||||||
|
|
||||||
|
|
||||||
def test_residue_incidents_in_uncovered_years_are_excluded(tmp_path):
|
|
||||||
# force-b stops publishing after 2023, but a force-a file contains a 2024
|
|
||||||
# incident that falls inside the b-postcode's buffer (cross-border residue,
|
|
||||||
# the Greater Manchester pattern). That incident must not produce a 2024
|
|
||||||
# bar for the b-postcode, nor leak into its pooled headline.
|
|
||||||
units = tmp_path / "units"
|
|
||||||
_write_boundaries(
|
|
||||||
units,
|
|
||||||
{
|
|
||||||
"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)],
|
|
||||||
"CD1": [_square_feature("CD1 1AA", 9000, 9000, 9010, 9010)],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
crime = tmp_path / "crime"
|
|
||||||
for month in range(1, 13):
|
|
||||||
ym23 = f"2023-{month:02d}"
|
|
||||||
ym24 = f"2024-{month:02d}"
|
|
||||||
_write_month(crime, ym23, [], force="force-a")
|
|
||||||
# b's own 2023 incidents establish force-b as its home force.
|
|
||||||
_write_month(
|
|
||||||
crime,
|
|
||||||
ym23,
|
|
||||||
[_crime_row(ym23, 9005, 9005, "Burglary")] if month <= 6 else [],
|
|
||||||
force="force-b",
|
|
||||||
)
|
|
||||||
# 2024: only force-a publishes; one of its incidents lands in CD1 1AA.
|
|
||||||
_write_month(
|
|
||||||
crime,
|
|
||||||
ym24,
|
|
||||||
[_crime_row(ym24, 9005, 9005, "Burglary")] if month == 1 else [],
|
|
||||||
force="force-a",
|
|
||||||
)
|
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
|
||||||
|
|
||||||
b_row = avg_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
|
|
||||||
# Pooled over force-b's 12 covered months (2023): 6 incidents -> 6/yr.
|
|
||||||
# The residue 2024 incident is excluded (force-b published 0 months in 2024).
|
|
||||||
assert b_row["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
|
||||||
|
|
||||||
b_by = by_year_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
|
|
||||||
bars = {p["year"]: p["count"] for p in b_by["Burglary (by year)"]}
|
|
||||||
assert set(bars) == {2023}
|
|
||||||
coverage = {c["year"]: c["months"] for c in b_by["covered_years"]}
|
|
||||||
assert coverage == {2023: 12}
|
|
||||||
|
|
||||||
|
|
||||||
def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
|
def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
|
||||||
# 2023 fully covered; 2024 has only 2 published months. With the default
|
# 2023 fully covered; 2024 has only 2 published months. With the default
|
||||||
# 6-month minimum, 2024 must produce neither a bar (annualising x6 charts
|
# 6-month minimum, 2024 must produce no bar -- but its incidents and months
|
||||||
# noise) nor a coverage entry -- but its incidents and months still count
|
# still count toward the pooled raw figure.
|
||||||
# toward the pooled headline.
|
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
|
|
@ -394,12 +340,10 @@ def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
|
||||||
ym = f"2024-{month:02d}"
|
ym = f"2024-{month:02d}"
|
||||||
_write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")])
|
_write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")])
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units)
|
||||||
|
|
||||||
# Pooled: 14 incidents over 14 covered months -> 12/yr.
|
# Pooled: 14 incidents over 14 covered months -> 12/yr.
|
||||||
assert avg_df.row(0, named=True)["Burglary (avg/yr)"] == pytest.approx(
|
assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
|
||||||
12.0, abs=0.05
|
|
||||||
)
|
|
||||||
row = by_year_df.row(0, named=True)
|
row = by_year_df.row(0, named=True)
|
||||||
bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]}
|
bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]}
|
||||||
assert set(bars) == {2023}
|
assert set(bars) == {2023}
|
||||||
|
|
@ -425,52 +369,119 @@ def test_by_year_output_is_dense_with_coverage(tmp_path):
|
||||||
crime = tmp_path / "crime"
|
crime = tmp_path / "crime"
|
||||||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
assert by_year_df.height == 2
|
assert by_year_df.height == 2
|
||||||
|
|
||||||
quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
||||||
assert quiet["Burglary (by year)"] is None
|
assert quiet["Burglary (by year)"] is None
|
||||||
assert [c["year"] for c in quiet["covered_years"]] == [2024]
|
assert [c["year"] for c in quiet["covered_years"]] == [2024]
|
||||||
# And the headline for the quiet postcode is a genuine 0, not null.
|
# The raw figure for the covered, crime-free postcode is a genuine 0, not null.
|
||||||
quiet_avg = avg_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
quiet_raw = raw_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
|
||||||
assert quiet_avg["Burglary (avg/yr)"] == 0.0
|
assert quiet_raw[_raw("Burglary")] == 0.0
|
||||||
|
|
||||||
|
|
||||||
def test_serious_rollup_avg_yr_equals_sum_of_components(tmp_path):
|
def test_serious_rollup_equals_sum_of_components(tmp_path):
|
||||||
# Burglary only in 2014, Robbery only in 2024 (one incident each, 2 covered
|
# Burglary only in 2023, Robbery only in 2024 (one incident each, 2 covered
|
||||||
# months total). Components pool over the same covered window (each
|
# months total, both inside the 7-year window). Components pool over the same
|
||||||
# 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
|
# covered window (each 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
)
|
)
|
||||||
|
|
||||||
crime = tmp_path / "crime"
|
crime = tmp_path / "crime"
|
||||||
_write_month(crime, "2014-01", [_crime_row("2014-01", 1005, 1005, "Burglary")])
|
_write_month(crime, "2023-01", [_crime_row("2023-01", 1005, 1005, "Burglary")])
|
||||||
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")])
|
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")])
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
|
|
||||||
avg = avg_df.row(0, named=True)
|
row = raw_df.row(0, named=True)
|
||||||
assert avg["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
assert row[_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
|
||||||
assert avg["Robbery (avg/yr)"] == pytest.approx(6.0, abs=0.05)
|
assert row[_raw("Robbery")] == pytest.approx(6.0, abs=0.05)
|
||||||
# Rollup == sum of its component (avg/yr) columns.
|
assert row[_raw("Serious crime")] == pytest.approx(12.0, abs=0.05)
|
||||||
assert avg["Serious crime (avg/yr)"] == pytest.approx(12.0, abs=0.05)
|
assert row[_raw("Serious crime")] == pytest.approx(
|
||||||
assert avg["Serious crime (avg/yr)"] == pytest.approx(
|
row[_raw("Burglary")] + row[_raw("Robbery")], abs=0.05
|
||||||
avg["Burglary (avg/yr)"] + avg["Robbery (avg/yr)"], abs=0.05
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# The by-year rollup series remains the per-year sum of the component bars.
|
|
||||||
serious_bars = {
|
serious_bars = {
|
||||||
p["year"]: p["count"]
|
p["year"]: p["count"]
|
||||||
for p in by_year_df.row(0, named=True)["Serious crime (by year)"]
|
for p in by_year_df.row(0, named=True)["Serious crime (by year)"]
|
||||||
}
|
}
|
||||||
assert serious_bars == {
|
assert serious_bars == {
|
||||||
2014: pytest.approx(12.0, abs=0.05),
|
2023: pytest.approx(12.0, abs=0.05),
|
||||||
2024: pytest.approx(12.0, abs=0.05),
|
2024: pytest.approx(12.0, abs=0.05),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_records_capture_each_counted_incident(tmp_path):
|
||||||
|
# Each (incident, postcode) match within the records window becomes a record
|
||||||
|
# row, carrying month/type/location/outcome/coords. A boundary incident
|
||||||
|
# counted for two postcodes appears once per postcode.
|
||||||
|
units = tmp_path / "units"
|
||||||
|
_write_boundaries(
|
||||||
|
units,
|
||||||
|
{
|
||||||
|
"AB1": [
|
||||||
|
_square_feature("AB1 1AA", 1000, 1000, 1010, 1010),
|
||||||
|
_square_feature("AB1 1AB", 1080, 1000, 1090, 1010),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
crime = tmp_path / "crime"
|
||||||
|
_write_month(
|
||||||
|
crime,
|
||||||
|
"2024-03",
|
||||||
|
[
|
||||||
|
# In the buffer overlap -> recorded for both postcodes.
|
||||||
|
_crime_row("2024-03", 1045, 1005, "Burglary", location="On or near High St", outcome="Under investigation"),
|
||||||
|
# Only in AB1 1AA's buffer; null outcome (police.uk leaves ASB blank).
|
||||||
|
_crime_row("2024-03", 1005, 1005, "Anti-social behaviour", location="On or near Mill Ln", outcome=""),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
|
|
||||||
|
assert set(records_df.columns) == {
|
||||||
|
"postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"
|
||||||
|
}
|
||||||
|
# Sorted by postcode.
|
||||||
|
assert records_df["postcode"].is_sorted()
|
||||||
|
# Burglary appears for BOTH postcodes (boundary multiplicity); ASB only for AA.
|
||||||
|
by_pc = records_df.group_by("postcode").agg(pl.col("crime_type").sort())
|
||||||
|
counts = {r["postcode"]: r["crime_type"] for r in by_pc.to_dicts()}
|
||||||
|
assert counts["AB1 1AA"] == ["Anti-social behaviour", "Burglary"]
|
||||||
|
assert counts["AB1 1AB"] == ["Burglary"]
|
||||||
|
# month_index = year*12 + (month-1) for 2024-03.
|
||||||
|
assert set(records_df["month_index"].to_list()) == {2024 * 12 + 2}
|
||||||
|
# Null outcome round-trips as null, not the string "".
|
||||||
|
asb = records_df.filter(pl.col("crime_type") == "Anti-social behaviour").row(0, named=True)
|
||||||
|
assert asb["outcome"] is None
|
||||||
|
assert asb["location"] == "On or near Mill Ln"
|
||||||
|
|
||||||
|
|
||||||
|
def test_records_window_aligns_to_the_headline_calendar_window(tmp_path):
|
||||||
|
# Records must cover exactly the longest (7y) headline window, which is
|
||||||
|
# calendar-year based. With a mid-year latest month (2025-06) the 7y window
|
||||||
|
# is calendar years 2019..2025, so an incident in 2018-09 -- which the
|
||||||
|
# headline excludes -- must also be excluded from the records, even though a
|
||||||
|
# naive rolling 84-month span (ending 2025-06) would wrongly include it. The
|
||||||
|
# first month of the earliest window year (2019-01) is kept.
|
||||||
|
units = tmp_path / "units"
|
||||||
|
_write_boundaries(
|
||||||
|
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
|
||||||
|
)
|
||||||
|
crime = tmp_path / "crime"
|
||||||
|
_write_month(crime, "2018-09", [_crime_row("2018-09", 1005, 1005, "Burglary")])
|
||||||
|
_write_month(crime, "2019-01", [_crime_row("2019-01", 1005, 1005, "Burglary")])
|
||||||
|
_write_month(crime, "2025-06", [_crime_row("2025-06", 1005, 1005, "Burglary")])
|
||||||
|
|
||||||
|
_, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
|
|
||||||
|
# 2018-09 (year*12+8) is in the rolling 84-month span but NOT the 7y calendar
|
||||||
|
# window, so it is excluded; 2019-01 and 2025-06 are kept.
|
||||||
|
assert set(records_df["month_index"].to_list()) == {2019 * 12 + 0, 2025 * 12 + 5}
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
|
def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
|
||||||
units = tmp_path / "units"
|
units = tmp_path / "units"
|
||||||
_write_boundaries(
|
_write_boundaries(
|
||||||
|
|
@ -487,11 +498,10 @@ def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
avg_df, _ = _run(tmp_path, crime, units)
|
raw_df, _, _ = _run(tmp_path, crime, units)
|
||||||
columns = avg_df.columns
|
columns = raw_df.columns
|
||||||
# The unknown type is dropped (no column for it) but a warning is emitted.
|
assert _raw("Cyber fraud") not in columns
|
||||||
assert "Cyber fraud (avg/yr)" not in columns
|
assert _raw("Burglary") in columns
|
||||||
assert "Burglary (avg/yr)" in columns
|
|
||||||
err = capsys.readouterr().err
|
err = capsys.readouterr().err
|
||||||
assert "Cyber fraud" in err
|
assert "Cyber fraud" in err
|
||||||
assert "WARNING" in err
|
assert "WARNING" in err
|
||||||
|
|
@ -515,11 +525,11 @@ def test_legacy_crime_types_are_mapped(tmp_path):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
|
raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
|
||||||
row = avg_df.to_dicts()[0]
|
row = raw_df.to_dicts()[0]
|
||||||
# Single postcode -> area-norm factor 1.0; single covered month -> x12.
|
# Single covered month (relative to a 2013-latest window) -> x12.
|
||||||
assert row["Violence and sexual offences (avg/yr)"] == 12.0
|
assert row[_raw("Violence and sexual offences")] == 12.0
|
||||||
assert row["Public order (avg/yr)"] == 12.0
|
assert row[_raw("Public order")] == 12.0
|
||||||
|
|
||||||
by_year_row = by_year_df.row(0, named=True)
|
by_year_row = by_year_df.row(0, named=True)
|
||||||
assert by_year_row["Violence and sexual offences (by year)"] == [
|
assert by_year_row["Violence and sexual offences (by year)"] == [
|
||||||
|
|
|
||||||
136
pipeline/transform/test_join_price_estimates.py
Normal file
136
pipeline/transform/test_join_price_estimates.py
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
"""Tests for joining slim price estimates back onto properties.parquet.
|
||||||
|
|
||||||
|
estimate.py emits (Postcode, coalesced address, estimate columns) and
|
||||||
|
join_estimates attaches them by that natural key. These tests pin the
|
||||||
|
properties that make the key safe: it maps estimates onto the right rows
|
||||||
|
regardless of order (a shuffled estimates frame is the worst case), it is
|
||||||
|
idempotent, and it refuses a partial/foreign estimates file rather than
|
||||||
|
silently nulling prices.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pipeline.transform.join_price_estimates import join_estimates
|
||||||
|
from pipeline.transform.price_estimation.utils import (
|
||||||
|
ESTIMATE_COLUMNS,
|
||||||
|
JOIN_ADDRESS,
|
||||||
|
JOIN_KEYS,
|
||||||
|
join_address_expr,
|
||||||
|
)
|
||||||
|
|
||||||
|
N = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _write_merged(path: Path) -> pl.DataFrame:
|
||||||
|
"""properties.parquet with the natural-key columns, a sentinel order column,
|
||||||
|
and no estimates. Half the rows are sale-addressed, half EPC-only, so the
|
||||||
|
coalesce in the key is exercised; every coalesced address is unique."""
|
||||||
|
df = pl.DataFrame(
|
||||||
|
{
|
||||||
|
"Postcode": [f"AA{i % 7} {i % 9}AA" for i in range(N)],
|
||||||
|
"Address per Property Register": [
|
||||||
|
f"reg-{i}" if i % 2 == 0 else None for i in range(N)
|
||||||
|
],
|
||||||
|
"Address per EPC": [f"epc-{i}" if i % 2 == 1 else None for i in range(N)],
|
||||||
|
"order": list(range(N)),
|
||||||
|
"junk": [f"x{i}" for i in range(N)],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
df.write_parquet(path)
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def _write_estimates(path: Path, merged_path: Path, *, shuffle: bool = True) -> None:
|
||||||
|
"""Estimates keyed by the natural key, derived from the merged file the way
|
||||||
|
estimate.py does. Estimate = order * 1000 so each row is checkable. Shuffled
|
||||||
|
by default to prove order-independence."""
|
||||||
|
est = (
|
||||||
|
pl.read_parquet(merged_path)
|
||||||
|
.with_columns(join_address_expr())
|
||||||
|
.with_columns(
|
||||||
|
(pl.col("order") * 1000).cast(pl.Float64).alias("Estimated current price"),
|
||||||
|
(pl.col("order") * 10).cast(pl.Int32).alias("Est. price per sqm"),
|
||||||
|
)
|
||||||
|
.select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
|
||||||
|
)
|
||||||
|
if shuffle:
|
||||||
|
est = est.sample(fraction=1.0, shuffle=True, seed=7)
|
||||||
|
est.write_parquet(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_join_attaches_estimates_to_the_right_rows(tmp_path: Path):
|
||||||
|
props = tmp_path / "properties.parquet"
|
||||||
|
estimates = tmp_path / "price_estimates.parquet"
|
||||||
|
_write_merged(props)
|
||||||
|
_write_estimates(estimates, props)
|
||||||
|
|
||||||
|
written = join_estimates(props, estimates)
|
||||||
|
out = pl.read_parquet(props)
|
||||||
|
|
||||||
|
assert written == N
|
||||||
|
assert out.height == N
|
||||||
|
# Order preserved and the address-half of the key is not left behind.
|
||||||
|
assert out["order"].to_list() == list(range(N))
|
||||||
|
assert out["junk"].to_list() == [f"x{i}" for i in range(N)]
|
||||||
|
assert JOIN_ADDRESS not in out.columns
|
||||||
|
# Every row carries its own estimate, matched by key despite the shuffle.
|
||||||
|
assert out["Estimated current price"].to_list() == [float(i * 1000) for i in range(N)]
|
||||||
|
assert out["Est. price per sqm"].to_list() == [i * 10 for i in range(N)]
|
||||||
|
assert out["Estimated current price"].null_count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_rerun_is_idempotent(tmp_path: Path):
|
||||||
|
props = tmp_path / "properties.parquet"
|
||||||
|
estimates = tmp_path / "price_estimates.parquet"
|
||||||
|
_write_merged(props)
|
||||||
|
_write_estimates(estimates, props)
|
||||||
|
|
||||||
|
join_estimates(props, estimates)
|
||||||
|
first = pl.read_parquet(props)
|
||||||
|
join_estimates(props, estimates) # second run on the augmented file
|
||||||
|
second = pl.read_parquet(props)
|
||||||
|
|
||||||
|
assert second.equals(first)
|
||||||
|
assert second.columns.count("Estimated current price") == 1
|
||||||
|
assert second.columns.count("Est. price per sqm") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_estimate_is_rejected(tmp_path: Path):
|
||||||
|
"""A property with no matching estimate (diverged dwelling universe) must
|
||||||
|
fail loudly rather than silently leave its price null."""
|
||||||
|
props = tmp_path / "properties.parquet"
|
||||||
|
estimates = tmp_path / "price_estimates.parquet"
|
||||||
|
_write_merged(props)
|
||||||
|
_write_estimates(estimates, props)
|
||||||
|
# Drop one estimate so a property key is no longer covered.
|
||||||
|
pl.read_parquet(estimates).head(N - 1).write_parquet(estimates)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="no matching estimate"):
|
||||||
|
join_estimates(props, estimates)
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_key_is_rejected(tmp_path: Path):
|
||||||
|
props = tmp_path / "properties.parquet"
|
||||||
|
estimates = tmp_path / "price_estimates.parquet"
|
||||||
|
_write_merged(props)
|
||||||
|
_write_estimates(estimates, props)
|
||||||
|
# Force row 1's key to collide with row 0's.
|
||||||
|
est = pl.read_parquet(estimates).sort("Estimated current price")
|
||||||
|
row0 = est.row(0, named=True)
|
||||||
|
est = est.with_columns(
|
||||||
|
pl.when(pl.int_range(pl.len()) == 1)
|
||||||
|
.then(pl.lit(row0["Postcode"]))
|
||||||
|
.otherwise(pl.col("Postcode"))
|
||||||
|
.alias("Postcode"),
|
||||||
|
pl.when(pl.int_range(pl.len()) == 1)
|
||||||
|
.then(pl.lit(row0[JOIN_ADDRESS]))
|
||||||
|
.otherwise(pl.col(JOIN_ADDRESS))
|
||||||
|
.alias(JOIN_ADDRESS),
|
||||||
|
)
|
||||||
|
est.write_parquet(estimates)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="not unique"):
|
||||||
|
join_estimates(props, estimates)
|
||||||
|
|
@ -10,14 +10,11 @@ from pipeline.transform.merge import (
|
||||||
LISTED_BUILDING_FEATURE,
|
LISTED_BUILDING_FEATURE,
|
||||||
TREE_DENSITY_FEATURE,
|
TREE_DENSITY_FEATURE,
|
||||||
_LISTING_OVERLAY_SOURCES,
|
_LISTING_OVERLAY_SOURCES,
|
||||||
_active_english_postcode_area,
|
|
||||||
_build_unmatched_listing_seed_rows,
|
_build_unmatched_listing_seed_rows,
|
||||||
_canonical_postcode_expr,
|
_canonical_postcode_expr,
|
||||||
_best_listing_match,
|
_best_listing_match,
|
||||||
_coalesce_direct_epc_columns,
|
_coalesce_direct_epc_columns,
|
||||||
_dedupe_collapsed_properties,
|
|
||||||
_fill_property_level_no_defaults,
|
_fill_property_level_no_defaults,
|
||||||
_filter_to_active_english_postcodes,
|
|
||||||
_join_area_side_tables,
|
_join_area_side_tables,
|
||||||
_finalize_listings,
|
_finalize_listings,
|
||||||
_integrate_listings,
|
_integrate_listings,
|
||||||
|
|
@ -31,7 +28,6 @@ from pipeline.transform.merge import (
|
||||||
_matched_listed_building_flags,
|
_matched_listed_building_flags,
|
||||||
_postcode_conservation_area_flags,
|
_postcode_conservation_area_flags,
|
||||||
_postcode_listed_building_candidates,
|
_postcode_listed_building_candidates,
|
||||||
_remap_terminated_postcodes,
|
|
||||||
_split_normal_outputs,
|
_split_normal_outputs,
|
||||||
_tree_density_by_postcode,
|
_tree_density_by_postcode,
|
||||||
_validate_lad_source_coverage,
|
_validate_lad_source_coverage,
|
||||||
|
|
@ -39,6 +35,12 @@ from pipeline.transform.merge import (
|
||||||
_validate_postcode_feature_output,
|
_validate_postcode_feature_output,
|
||||||
_validate_property_postcodes,
|
_validate_property_postcodes,
|
||||||
)
|
)
|
||||||
|
from pipeline.transform.property_base import (
|
||||||
|
_active_english_postcode_area,
|
||||||
|
_dedupe_collapsed_properties,
|
||||||
|
_filter_to_active_english_postcodes,
|
||||||
|
_remap_terminated_postcodes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
|
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
|
||||||
|
|
@ -115,13 +117,16 @@ def test_tree_density_is_area_level_and_survives_the_split() -> None:
|
||||||
assert TREE_DENSITY_FEATURE not in properties_df.columns
|
assert TREE_DENSITY_FEATURE not in properties_df.columns
|
||||||
|
|
||||||
|
|
||||||
def test_crime_columns_are_spatial_counts_not_per_capita() -> None:
|
def test_crime_columns_are_average_annual_counts() -> None:
|
||||||
# Crime is now a raw spatial count per postcode; the per-1k-residents
|
# Crime is the average annual recorded incident count (incidents/yr) over
|
||||||
# variants were dropped along with the LSOA population denominator.
|
# 7-year and 2-year windows; the old per-1,000 "(per 1k/yr, …)" rate columns
|
||||||
assert "Serious crime (avg/yr)" in _AREA_COLUMNS
|
# are gone.
|
||||||
assert "Minor crime (avg/yr)" in _AREA_COLUMNS
|
assert "Serious crime (/yr, 7y)" in _AREA_COLUMNS
|
||||||
assert "Serious crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
|
assert "Serious crime (/yr, 2y)" in _AREA_COLUMNS
|
||||||
assert "Minor crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
|
assert "Minor crime (/yr, 7y)" in _AREA_COLUMNS
|
||||||
|
assert "Burglary (/yr, 2y)" in _AREA_COLUMNS
|
||||||
|
assert "Serious crime (avg/yr)" not in _AREA_COLUMNS
|
||||||
|
assert "Minor crime (avg/yr)" not in _AREA_COLUMNS
|
||||||
|
|
||||||
|
|
||||||
def test_active_english_postcode_area_filters_to_active_england() -> None:
|
def test_active_english_postcode_area_filters_to_active_england() -> None:
|
||||||
|
|
@ -292,8 +297,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
|
||||||
crime = pl.LazyFrame(
|
crime = pl.LazyFrame(
|
||||||
{
|
{
|
||||||
"postcode": ["AA1 1AA", "BB2 2BB"],
|
"postcode": ["AA1 1AA", "BB2 2BB"],
|
||||||
"Serious crime (avg/yr)": [1.0, 2.0],
|
"Serious crime (/yr, 7y)": [1.0, 2.0],
|
||||||
"Minor crime (avg/yr)": [3.0, 4.0],
|
"Minor crime (/yr, 7y)": [3.0, 4.0],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
joined = _join_area_side_tables(
|
joined = _join_area_side_tables(
|
||||||
|
|
@ -343,8 +348,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
|
||||||
crime = pl.LazyFrame(
|
crime = pl.LazyFrame(
|
||||||
{
|
{
|
||||||
"postcode": ["AB1 2CD", "EF3 4GH"],
|
"postcode": ["AB1 2CD", "EF3 4GH"],
|
||||||
"Serious crime (avg/yr)": [1.0, 2.0],
|
"Serious crime (/yr, 7y)": [1.0, 2.0],
|
||||||
"Minor crime (avg/yr)": [3.0, 4.0],
|
"Minor crime (/yr, 7y)": [3.0, 4.0],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
|
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
|
||||||
|
|
@ -1314,28 +1319,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
|
||||||
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
|
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
|
||||||
|
|
||||||
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
|
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
|
||||||
# rollup headlines are precomputed values (deliberately NOT the per-type sum,
|
# rollup rate columns are precomputed in crime_spatial and read straight
|
||||||
# which would be 10.0 each) so this test proves the merge consumes the
|
# through unchanged (the merge no longer renames or re-sums them).
|
||||||
# precomputed column rather than re-summing per-type columns.
|
|
||||||
crime = pl.LazyFrame(
|
crime = pl.LazyFrame(
|
||||||
{
|
{
|
||||||
"postcode": ["AA1 1AA"],
|
"postcode": ["AA1 1AA"],
|
||||||
"Violence and sexual offences (avg/yr)": [1.0],
|
"Burglary (/yr, 7y)": [3.0],
|
||||||
"Robbery (avg/yr)": [2.0],
|
"Burglary (/yr, 2y)": [3.5],
|
||||||
"Burglary (avg/yr)": [3.0],
|
"Serious crime (/yr, 7y)": [7.5],
|
||||||
"Possession of weapons (avg/yr)": [4.0],
|
"Serious crime (/yr, 2y)": [8.0],
|
||||||
"Anti-social behaviour (avg/yr)": [1.0],
|
"Minor crime (/yr, 7y)": [4.2],
|
||||||
"Criminal damage and arson (avg/yr)": [1.0],
|
"Minor crime (/yr, 2y)": [4.6],
|
||||||
"Shoplifting (avg/yr)": [1.0],
|
|
||||||
"Bicycle theft (avg/yr)": [1.0],
|
|
||||||
"Theft from the person (avg/yr)": [1.0],
|
|
||||||
"Other theft (avg/yr)": [1.0],
|
|
||||||
"Vehicle crime (avg/yr)": [1.0],
|
|
||||||
"Public order (avg/yr)": [1.0],
|
|
||||||
"Drugs (avg/yr)": [1.0],
|
|
||||||
"Other crime (avg/yr)": [1.0],
|
|
||||||
"Serious crime (avg/yr)": [7.5],
|
|
||||||
"Minor crime (avg/yr)": [4.2],
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -1364,16 +1358,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
|
||||||
by_postcode = {
|
by_postcode = {
|
||||||
row["postcode"]: row
|
row["postcode"]: row
|
||||||
for row in joined.select(
|
for row in joined.select(
|
||||||
"postcode", "serious_crime_avg_yr", "minor_crime_avg_yr"
|
"postcode",
|
||||||
|
"Serious crime (/yr, 7y)",
|
||||||
|
"Minor crime (/yr, 2y)",
|
||||||
).iter_rows(named=True)
|
).iter_rows(named=True)
|
||||||
}
|
}
|
||||||
# Present postcode: rollups are the precomputed headline values, read through
|
# Present postcode: rollup rates pass through unchanged.
|
||||||
# unchanged (NOT the per-type sum of 10.0).
|
assert by_postcode["AA1 1AA"]["Serious crime (/yr, 7y)"] == 7.5
|
||||||
assert by_postcode["AA1 1AA"]["serious_crime_avg_yr"] == 7.5
|
assert by_postcode["AA1 1AA"]["Minor crime (/yr, 2y)"] == 4.6
|
||||||
assert by_postcode["AA1 1AA"]["minor_crime_avg_yr"] == 4.2
|
# Missing postcode: rates stay null rather than fabricating 0.0.
|
||||||
# Missing postcode: rollups stay null rather than fabricating 0.0.
|
assert by_postcode["BB2 2BB"]["Serious crime (/yr, 7y)"] is None
|
||||||
assert by_postcode["BB2 2BB"]["serious_crime_avg_yr"] is None
|
assert by_postcode["BB2 2BB"]["Minor crime (/yr, 2y)"] is None
|
||||||
assert by_postcode["BB2 2BB"]["minor_crime_avg_yr"] is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
|
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
mod actual_listings;
|
mod actual_listings;
|
||||||
pub mod area_crime_averages;
|
pub mod area_crime_averages;
|
||||||
pub mod crime_by_year;
|
pub mod crime_by_year;
|
||||||
|
pub mod crime_records;
|
||||||
mod developments;
|
mod developments;
|
||||||
mod places;
|
mod places;
|
||||||
mod poi;
|
mod poi;
|
||||||
|
|
@ -65,6 +66,7 @@ where
|
||||||
pub use actual_listings::{ActualListing, ActualListingData};
|
pub use actual_listings::{ActualListing, ActualListingData};
|
||||||
pub use area_crime_averages::AreaCrimeAverages;
|
pub use area_crime_averages::AreaCrimeAverages;
|
||||||
pub use crime_by_year::CrimeByYearData;
|
pub use crime_by_year::CrimeByYearData;
|
||||||
|
pub use crime_records::CrimeRecords;
|
||||||
pub use developments::{DevelopmentData, DevelopmentSite};
|
pub use developments::{DevelopmentData, DevelopmentSite};
|
||||||
pub use places::{
|
pub use places::{
|
||||||
compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,
|
compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,
|
||||||
|
|
|
||||||
|
|
@ -1,41 +1,61 @@
|
||||||
//! Precomputed per-outcode and per-postcode-sector average crime rates.
|
//! Precomputed per-outcode and per-postcode-sector average crime counts.
|
||||||
//!
|
//!
|
||||||
//! The right pane shows each crime metric's national average (the global
|
//! The right pane shows each crime metric's national average (the global
|
||||||
//! feature-histogram mean). To let users see how an area compares with its
|
//! feature-histogram mean). To let users see how an area compares with its
|
||||||
//! immediate surroundings, we also precompute the mean headline crime rate
|
//! immediate surroundings, we also show the mean average-annual crime count
|
||||||
//! (`"X (avg/yr)"`) across every property in the selection's outcode (e.g.
|
//! (`"X (/yr, 7y|2y)"`) across every property in the selection's outcode
|
||||||
//! `"E14"`) and postcode sector (e.g. `"E14 2"`).
|
//! (e.g. `"E14"`) and postcode sector (e.g. `"E14 2"`).
|
||||||
//!
|
//!
|
||||||
//! Crime figures are constant within a postcode (the pipeline merges them on
|
//! These averages are precomputed by the data pipeline
|
||||||
//! the postcode key), so each postcode's value is read once — from its first
|
//! (`pipeline/transform/area_crime_averages.py`) and loaded here from a side
|
||||||
//! row — and property-weighted by the postcode's row count. That keeps these
|
//! parquet. Crime figures are constant within a postcode, so the pipeline
|
||||||
//! averages on the same property-weighted basis as the national average, so the
|
//! property-weights each postcode's value by its property count — keeping these
|
||||||
|
//! averages on the same property-weighted basis as the per-selection mean, so the
|
||||||
//! four numbers (this area / sector / outcode / nation) are directly comparable.
|
//! four numbers (this area / sector / outcode / nation) are directly comparable.
|
||||||
|
|
||||||
use rustc_hash::FxHashMap;
|
use std::path::Path;
|
||||||
|
|
||||||
/// Crime-feature name suffix that marks an annualised headline-rate column
|
use anyhow::{bail, Context};
|
||||||
/// (e.g. `"Burglary (avg/yr)"`). Stripped to derive the bare type name.
|
use polars::prelude::PlRefPath;
|
||||||
pub const AVG_YR_SUFFIX: &str = " (avg/yr)";
|
use polars::prelude::*;
|
||||||
|
use rustc_hash::FxHashMap;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
use super::run_polars_io;
|
||||||
|
|
||||||
|
/// Marker that identifies an average-annual crime-count column (e.g.
|
||||||
|
/// `"Burglary (/yr, 7y)"`). These are the filterable, area-comparable figures.
|
||||||
|
/// The full column name is kept as the key, so each per-area mean aligns with the
|
||||||
|
/// feature the frontend requests.
|
||||||
|
pub const COUNT_MARKER: &str = " (/yr, ";
|
||||||
|
|
||||||
|
/// `scope` column discriminator values written by the pipeline.
|
||||||
|
const SCOPE_NATIONAL: &str = "national";
|
||||||
|
const SCOPE_OUTCODE: &str = "outcode";
|
||||||
|
const SCOPE_SECTOR: &str = "sector";
|
||||||
|
|
||||||
pub struct AreaCrimeAverages {
|
pub struct AreaCrimeAverages {
|
||||||
/// Bare crime-type names (suffix stripped, e.g. `"Burglary"`), index-aligned
|
/// Full crime feature names (e.g. `"Burglary (/yr, 7y)"`), index-aligned with
|
||||||
/// with the per-area mean vectors. Matches `CrimeYearStats.name`.
|
/// the per-area mean vectors. Matches the feature names the frontend requests,
|
||||||
|
/// so each NumberLine can look its average up directly.
|
||||||
pub crime_types: Vec<String>,
|
pub crime_types: Vec<String>,
|
||||||
/// National mean headline rate per crime type (index-aligned with
|
/// National mean headline count per crime type (index-aligned with
|
||||||
/// `crime_types`). An EXACT property-weighted mean over every postcode, so it
|
/// `crime_types`). An EXACT property-weighted mean over every postcode, so it
|
||||||
/// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
|
/// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
|
||||||
/// unlike the histogram-bin national average, which is biased upward for the
|
/// unlike the histogram-bin national average, which is biased upward for the
|
||||||
/// right-skewed crime densities. `NaN` where no postcode has data.
|
/// right-skewed crime counts. `NaN` where no postcode has data.
|
||||||
pub national: Vec<f32>,
|
pub national: Vec<f32>,
|
||||||
/// Outcode (e.g. `"E14"`) → mean headline rate per crime type. `NaN` where
|
/// Outcode (e.g. `"E14"`) → mean headline count per crime type. `NaN` where
|
||||||
/// the outcode has no data for that type.
|
/// the outcode has no data for that type.
|
||||||
pub by_outcode: FxHashMap<String, Vec<f32>>,
|
pub by_outcode: FxHashMap<String, Vec<f32>>,
|
||||||
/// Postcode sector (e.g. `"E14 2"`) → mean headline rate per crime type.
|
/// Postcode sector (e.g. `"E14 2"`) → mean headline count per crime type.
|
||||||
pub by_sector: FxHashMap<String, Vec<f32>>,
|
pub by_sector: FxHashMap<String, Vec<f32>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AreaCrimeAverages {
|
impl AreaCrimeAverages {
|
||||||
|
/// Empty table — used only by the test-only `AppState` builder (the real
|
||||||
|
/// server always loads the precomputed parquet).
|
||||||
|
#[cfg(test)]
|
||||||
pub fn empty() -> Self {
|
pub fn empty() -> Self {
|
||||||
Self {
|
Self {
|
||||||
crime_types: Vec::new(),
|
crime_types: Vec::new(),
|
||||||
|
|
@ -44,4 +64,202 @@ impl AreaCrimeAverages {
|
||||||
by_sector: FxHashMap::default(),
|
by_sector: FxHashMap::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||||
|
run_polars_io(|| Self::load_inner(path))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_inner(path: &Path) -> anyhow::Result<Self> {
|
||||||
|
info!("Loading area crime averages from {}", path.display());
|
||||||
|
let pl_path = PlRefPath::try_from_path(path).with_context(|| {
|
||||||
|
format!(
|
||||||
|
"Failed to normalize area-crime-averages parquet path {}",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let df = LazyFrame::scan_parquet(pl_path, Default::default())
|
||||||
|
.with_context(|| {
|
||||||
|
format!(
|
||||||
|
"Failed to scan area-crime-averages parquet at {}",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
.collect()
|
||||||
|
.with_context(|| {
|
||||||
|
format!(
|
||||||
|
"Failed to read area-crime-averages parquet at {}",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Crime columns are those carrying the count marker; their full names
|
||||||
|
// are kept (in column order) as the keys, so every per-area mean vector is
|
||||||
|
// index-aligned with `crime_types` and matches the requested feature name.
|
||||||
|
let crime_cols: Vec<String> = df
|
||||||
|
.get_column_names()
|
||||||
|
.iter()
|
||||||
|
.filter(|name| name.contains(COUNT_MARKER))
|
||||||
|
.map(|name| name.to_string())
|
||||||
|
.collect();
|
||||||
|
if crime_cols.is_empty() {
|
||||||
|
bail!(
|
||||||
|
"area-crime-averages parquet at {} has no '*{COUNT_MARKER}*' count columns",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let crime_types: Vec<String> = crime_cols.clone();
|
||||||
|
let n = crime_cols.len();
|
||||||
|
|
||||||
|
let scope_col = df
|
||||||
|
.column("scope")
|
||||||
|
.context("area-crime-averages parquet missing 'scope' column")?
|
||||||
|
.str()
|
||||||
|
.context("'scope' column is not a string")?;
|
||||||
|
let area_col = df
|
||||||
|
.column("area")
|
||||||
|
.context("area-crime-averages parquet missing 'area' column")?
|
||||||
|
.str()
|
||||||
|
.context("'area' column is not a string")?;
|
||||||
|
|
||||||
|
// Hold the casts alive while we borrow `Float32Chunked` views into them.
|
||||||
|
let casts: Vec<Column> = crime_cols
|
||||||
|
.iter()
|
||||||
|
.map(|name| {
|
||||||
|
df.column(name)
|
||||||
|
.and_then(|col| col.cast(&DataType::Float32))
|
||||||
|
.with_context(|| format!("Failed to read crime column '{name}' as f32"))
|
||||||
|
})
|
||||||
|
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||||
|
let crime_views: Vec<&Float32Chunked> = casts
|
||||||
|
.iter()
|
||||||
|
.zip(crime_cols.iter())
|
||||||
|
.map(|(col, name)| {
|
||||||
|
col.f32()
|
||||||
|
.with_context(|| format!("crime column '{name}' is not f32 after cast"))
|
||||||
|
})
|
||||||
|
.collect::<anyhow::Result<Vec<_>>>()?;
|
||||||
|
|
||||||
|
let read_values = |row: usize| -> Vec<f32> {
|
||||||
|
crime_views
|
||||||
|
.iter()
|
||||||
|
.map(|view| view.get(row).unwrap_or(f32::NAN))
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut national: Option<Vec<f32>> = None;
|
||||||
|
let mut by_outcode: FxHashMap<String, Vec<f32>> = FxHashMap::default();
|
||||||
|
let mut by_sector: FxHashMap<String, Vec<f32>> = FxHashMap::default();
|
||||||
|
|
||||||
|
for row in 0..df.height() {
|
||||||
|
let scope = scope_col
|
||||||
|
.get(row)
|
||||||
|
.with_context(|| format!("area-crime-averages row {row} has null scope"))?;
|
||||||
|
match scope {
|
||||||
|
SCOPE_NATIONAL => national = Some(read_values(row)),
|
||||||
|
SCOPE_OUTCODE => {
|
||||||
|
if let Some(area) = area_col.get(row) {
|
||||||
|
by_outcode.insert(area.to_string(), read_values(row));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SCOPE_SECTOR => {
|
||||||
|
if let Some(area) = area_col.get(row) {
|
||||||
|
by_sector.insert(area.to_string(), read_values(row));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other => bail!("area-crime-averages row {row} has unknown scope '{other}'"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let national = national.context(
|
||||||
|
"area-crime-averages parquet has no 'national' row; regenerate it with \
|
||||||
|
pipeline.transform.area_crime_averages",
|
||||||
|
)?;
|
||||||
|
if national.len() != n {
|
||||||
|
bail!(
|
||||||
|
"area-crime-averages national row has {} values, expected {n}",
|
||||||
|
national.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
info!(
|
||||||
|
outcodes = by_outcode.len(),
|
||||||
|
sectors = by_sector.len(),
|
||||||
|
crime_types = crime_types.len(),
|
||||||
|
"Area crime averages loaded"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
crime_types,
|
||||||
|
national,
|
||||||
|
by_outcode,
|
||||||
|
by_sector,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn write_fixture(path: &Path) {
|
||||||
|
// national + one outcode (E14) + one sector (E14 2). Robbery is null for
|
||||||
|
// the outcode to exercise the NaN round-trip.
|
||||||
|
let mut df = df!(
|
||||||
|
"scope" => ["national", "outcode", "sector"],
|
||||||
|
"area" => ["", "E14", "E14 2"],
|
||||||
|
"Burglary (/yr, 7y)" => [8.75f32, 12.5, 12.5],
|
||||||
|
"Robbery (/yr, 7y)" => [Some(1.43f32), None, Some(2.0)],
|
||||||
|
// A non-crime column must be ignored by the loader.
|
||||||
|
"Median age" => [40.0f32, 41.0, 42.0],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut file = std::fs::File::create(path).unwrap();
|
||||||
|
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_round_trips_national_outcode_sector() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("acavg-{}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let path = dir.join("area_crime_averages.parquet");
|
||||||
|
write_fixture(&path);
|
||||||
|
|
||||||
|
let avgs = AreaCrimeAverages::load(&path).unwrap();
|
||||||
|
|
||||||
|
// Crime columns are discovered by the count marker; "Median age" is not.
|
||||||
|
assert_eq!(
|
||||||
|
avgs.crime_types,
|
||||||
|
vec!["Burglary (/yr, 7y)", "Robbery (/yr, 7y)"]
|
||||||
|
);
|
||||||
|
assert_eq!(avgs.national, vec![8.75, 1.43]);
|
||||||
|
|
||||||
|
let e14 = avgs.by_outcode.get("E14").unwrap();
|
||||||
|
assert_eq!(e14[0], 12.5);
|
||||||
|
// The null robbery value becomes NaN, which the consumer drops to None.
|
||||||
|
assert!(e14[1].is_nan());
|
||||||
|
|
||||||
|
let e14_2 = avgs.by_sector.get("E14 2").unwrap();
|
||||||
|
assert_eq!(e14_2, &vec![12.5, 2.0]);
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).ok();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_rejects_parquet_without_national_row() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("acavg-nonat-{}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let path = dir.join("no_national.parquet");
|
||||||
|
let mut df = df!(
|
||||||
|
"scope" => ["outcode"],
|
||||||
|
"area" => ["E14"],
|
||||||
|
"Burglary (/yr, 7y)" => [12.5f32],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut file = std::fs::File::create(&path).unwrap();
|
||||||
|
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
|
||||||
|
|
||||||
|
assert!(AreaCrimeAverages::load(&path).is_err());
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).ok();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
534
server-rs/src/data/crime_records.rs
Normal file
534
server-rs/src/data/crime_records.rs
Normal file
|
|
@ -0,0 +1,534 @@
|
||||||
|
//! Individual police.uk crime records (last 7 years) backing the right pane's
|
||||||
|
//! "individual crimes" list and the `/api/crime-records` endpoint.
|
||||||
|
//!
|
||||||
|
//! This table is enormous — ~500M rows, because each incident is replicated to
|
||||||
|
//! every postcode whose buffer covers it (see [`gather`](CrimeRecords::gather)),
|
||||||
|
//! so it is NOT held as a `Vec<struct>`: each field is a flat columnar
|
||||||
|
//! [`SpillVec`] (mmap-backed and kernel-reclaimable when `--spill-dir` is set),
|
||||||
|
//! small string fields are dictionary-encoded, and the parquet is pre-sorted by
|
||||||
|
//! postcode so each postcode's records are a contiguous `[start, start+count)`
|
||||||
|
//! slice located via a CSR-style offset index. Resident RSS is ~0 until records
|
||||||
|
//! are actually read.
|
||||||
|
//!
|
||||||
|
//! At ~500M rows the parquet's string columns (postcode/type/location/outcome)
|
||||||
|
//! decode to tens of GB if read whole, so the loader never materialises the
|
||||||
|
//! whole `DataFrame`: it streams the file in bounded row-count chunks (only the
|
||||||
|
//! row groups overlapping each slice are decoded) and writes each column
|
||||||
|
//! straight into its (optionally spilled) backing store via [`SpillVecBuilder`],
|
||||||
|
//! keeping the transient footprint to one chunk plus the index maps.
|
||||||
|
|
||||||
|
use std::fs::File;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use anyhow::{bail, Context};
|
||||||
|
use lasso::{Rodeo, RodeoReader, Spur};
|
||||||
|
use polars::prelude::*;
|
||||||
|
use rustc_hash::FxHashMap;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
use super::run_polars_io;
|
||||||
|
use super::spill::{SpillVec, SpillVecBuilder};
|
||||||
|
|
||||||
|
/// Rows decoded per streaming slice. `with_slice` decodes only the row groups
|
||||||
|
/// overlapping the slice, so the transient decode is roughly one chunk's worth
|
||||||
|
/// (~tens of MB at the writer's ~123k-row groups) instead of the tens-of-GB
|
||||||
|
/// whole-file `DataFrame`. The bound's only dependency on file layout is a
|
||||||
|
/// reasonable input row-group size, which our pipeline writer produces.
|
||||||
|
const CHUNK_ROWS: usize = 2_000_000;
|
||||||
|
|
||||||
|
/// A resolved view of one record (strings dereferenced from the dictionaries).
|
||||||
|
pub struct CrimeRecordView<'a> {
|
||||||
|
/// `year * 12 + (month - 1)`.
|
||||||
|
pub month_index: u32,
|
||||||
|
pub crime_type: &'a str,
|
||||||
|
pub outcome: Option<&'a str>,
|
||||||
|
pub location: Option<&'a str>,
|
||||||
|
pub lat: f32,
|
||||||
|
pub lon: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CrimeRecords {
|
||||||
|
month: SpillVec<u32>,
|
||||||
|
ctype: SpillVec<u8>,
|
||||||
|
outcome: SpillVec<u8>,
|
||||||
|
location: SpillVec<Spur>,
|
||||||
|
lat: SpillVec<f32>,
|
||||||
|
lon: SpillVec<f32>,
|
||||||
|
/// Dictionary for `ctype` (bare crime type names, e.g. "Burglary").
|
||||||
|
crime_type_dict: Vec<String>,
|
||||||
|
/// Dictionary for `outcome`; index 0 is the empty/unknown sentinel.
|
||||||
|
outcome_dict: Vec<String>,
|
||||||
|
/// Resolver for the interned `location` strings (`""` means withheld).
|
||||||
|
location_resolver: RodeoReader,
|
||||||
|
/// Postcode → `(start, count)` into the columnar arrays (records for a
|
||||||
|
/// postcode are contiguous because the parquet is sorted by postcode).
|
||||||
|
by_postcode: FxHashMap<String, (u32, u32)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CrimeRecords {
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
month: SpillVec::owned(Vec::new()),
|
||||||
|
ctype: SpillVec::owned(Vec::new()),
|
||||||
|
outcome: SpillVec::owned(Vec::new()),
|
||||||
|
location: SpillVec::owned(Vec::new()),
|
||||||
|
lat: SpillVec::owned(Vec::new()),
|
||||||
|
lon: SpillVec::owned(Vec::new()),
|
||||||
|
crime_type_dict: Vec::new(),
|
||||||
|
outcome_dict: vec![String::new()],
|
||||||
|
location_resolver: Rodeo::default().into_reader(),
|
||||||
|
by_postcode: FxHashMap::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of records stored for a postcode (0 if none).
|
||||||
|
pub fn total_for(&self, postcode: &str) -> u32 {
|
||||||
|
self.by_postcode.get(postcode).map_or(0, |&(_, c)| c)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve a record index to a borrowing view.
|
||||||
|
pub fn view(&self, idx: u32) -> CrimeRecordView<'_> {
|
||||||
|
let i = idx as usize;
|
||||||
|
let outcome_idx = self.outcome[i] as usize;
|
||||||
|
let outcome = self
|
||||||
|
.outcome_dict
|
||||||
|
.get(outcome_idx)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(String::as_str);
|
||||||
|
let location = self.location_resolver.resolve(&self.location[i]);
|
||||||
|
CrimeRecordView {
|
||||||
|
month_index: self.month[i],
|
||||||
|
crime_type: self
|
||||||
|
.crime_type_dict
|
||||||
|
.get(self.ctype[i] as usize)
|
||||||
|
.map_or("", String::as_str),
|
||||||
|
outcome,
|
||||||
|
location: (!location.is_empty()).then_some(location),
|
||||||
|
lat: self.lat[i],
|
||||||
|
lon: self.lon[i],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record indices across `postcodes`, newest first, optionally restricted to
|
||||||
|
/// months `>= since_month`. These are exactly the incidents counted for the
|
||||||
|
/// selected postcodes — for a single postcode that is its precise incident
|
||||||
|
/// list; for a multi-postcode selection a boundary incident counted for
|
||||||
|
/// several postcodes appears once per postcode, matching the count. We do not
|
||||||
|
/// de-duplicate because police.uk snaps many genuinely distinct incidents
|
||||||
|
/// (especially anti-social behaviour) to the same point/month and provides no
|
||||||
|
/// per-incident id to tell a true duplicate from two real incidents apart.
|
||||||
|
pub fn gather(&self, postcodes: &[&str], since_month: Option<u32>) -> Vec<u32> {
|
||||||
|
let month = self.month.as_slice();
|
||||||
|
let mut out: Vec<u32> = Vec::new();
|
||||||
|
for pc in postcodes {
|
||||||
|
let Some(&(start, count)) = self.by_postcode.get(*pc) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for i in start..start + count {
|
||||||
|
if since_month.map_or(true, |s| month[i as usize] >= s) {
|
||||||
|
out.push(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.sort_unstable_by(|&a, &b| month[b as usize].cmp(&month[a as usize]));
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn load(path: &Path, spill_dir: Option<&Path>) -> anyhow::Result<Self> {
|
||||||
|
run_polars_io(|| Self::load_inner(path, spill_dir, CHUNK_ROWS))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_inner(
|
||||||
|
path: &Path,
|
||||||
|
spill_dir: Option<&Path>,
|
||||||
|
chunk_rows: usize,
|
||||||
|
) -> anyhow::Result<Self> {
|
||||||
|
// A zero chunk size would loop forever; the public entry point passes the
|
||||||
|
// const, but tests parameterise this to exercise chunk boundaries.
|
||||||
|
let chunk_rows = chunk_rows.max(1);
|
||||||
|
info!("Loading crime records from {}", path.display());
|
||||||
|
|
||||||
|
// Read the footer once for the row count, and keep it to hand to every
|
||||||
|
// per-chunk reader so the 2.9MB metadata is never re-parsed.
|
||||||
|
let metadata = {
|
||||||
|
let file = File::open(path).with_context(|| {
|
||||||
|
format!("Failed to open crime-records parquet at {}", path.display())
|
||||||
|
})?;
|
||||||
|
ParquetReader::new(file)
|
||||||
|
.get_metadata()
|
||||||
|
.with_context(|| {
|
||||||
|
format!("Failed to read crime-records parquet metadata at {}", path.display())
|
||||||
|
})?
|
||||||
|
.clone()
|
||||||
|
};
|
||||||
|
let n = metadata.num_rows;
|
||||||
|
// Record indices are stored as `u32` (and `by_postcode` holds `(start,
|
||||||
|
// count)` as `u32`), so the table must fit in that index space.
|
||||||
|
if n > u32::MAX as usize {
|
||||||
|
bail!("crime-records parquet has {n} rows, exceeding the u32 record-index limit");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Columns that, when spilling, are written straight into mmap-backed files
|
||||||
|
// as we stream — so the ~9GB of columnar data never lands on the heap.
|
||||||
|
let mut month = SpillVecBuilder::<u32>::with_len(n, spill_dir, "crime_month")?;
|
||||||
|
let mut ctype = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_ctype")?;
|
||||||
|
let mut outcome = SpillVecBuilder::<u8>::with_len(n, spill_dir, "crime_outcome")?;
|
||||||
|
let mut location = SpillVecBuilder::<Spur>::with_len(n, spill_dir, "crime_location")?;
|
||||||
|
let mut lat = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lat")?;
|
||||||
|
let mut lon = SpillVecBuilder::<f32>::with_len(n, spill_dir, "crime_lon")?;
|
||||||
|
|
||||||
|
let mut crime_type_dict: Vec<String> = Vec::new();
|
||||||
|
let mut type_index: FxHashMap<String, u8> = FxHashMap::default();
|
||||||
|
// Outcome index 0 is the empty/unknown sentinel.
|
||||||
|
let mut outcome_dict: Vec<String> = vec![String::new()];
|
||||||
|
let mut outcome_index: FxHashMap<String, u8> = FxHashMap::default();
|
||||||
|
let mut rodeo = Rodeo::default();
|
||||||
|
let empty_spur = rodeo.get_or_intern("");
|
||||||
|
|
||||||
|
let mut by_postcode: FxHashMap<String, (u32, u32)> = FxHashMap::default();
|
||||||
|
let mut cur_pc: Option<String> = None;
|
||||||
|
let mut cur_start: u32 = 0;
|
||||||
|
// Absolute record index across all chunks; drives both the CSR index and
|
||||||
|
// the column builders' write order.
|
||||||
|
let mut global_row: u32 = 0;
|
||||||
|
|
||||||
|
let columns: Vec<String> = ["postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"]
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut offset = 0usize;
|
||||||
|
while offset < n {
|
||||||
|
let len = chunk_rows.min(n - offset);
|
||||||
|
let file = File::open(path).with_context(|| {
|
||||||
|
format!("Failed to open crime-records parquet at {}", path.display())
|
||||||
|
})?;
|
||||||
|
let mut reader = ParquetReader::new(file);
|
||||||
|
reader.set_metadata(metadata.clone());
|
||||||
|
// `with_slice` only decodes the row groups overlapping `[offset,
|
||||||
|
// offset+len)` (the file is memory-mapped, so untouched groups are
|
||||||
|
// never faulted in), capping the transient decode to one chunk.
|
||||||
|
let df = reader
|
||||||
|
.with_columns(Some(columns.clone()))
|
||||||
|
.with_slice(Some((offset, len)))
|
||||||
|
.finish()
|
||||||
|
.with_context(|| {
|
||||||
|
format!(
|
||||||
|
"Failed to read crime-records rows [{offset}, {}) from {}",
|
||||||
|
offset + len,
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let postcode_col = df
|
||||||
|
.column("postcode")
|
||||||
|
.context("crime-records parquet missing 'postcode'")?
|
||||||
|
.str()
|
||||||
|
.context("'postcode' is not a string")?;
|
||||||
|
let month_col = df
|
||||||
|
.column("month_index")
|
||||||
|
.context("crime-records parquet missing 'month_index'")?
|
||||||
|
.cast(&DataType::Int32)
|
||||||
|
.context("'month_index' not castable to i32")?;
|
||||||
|
let month_ca = month_col.i32().context("'month_index' is not i32")?;
|
||||||
|
// Months are `year*12 + month0` (~24_000), always positive. A null or
|
||||||
|
// non-positive value means a corrupt parquet; fail loudly rather than
|
||||||
|
// silently clamping it to 0 and later rendering it as "0000-01".
|
||||||
|
if month_ca.null_count() > 0 {
|
||||||
|
bail!("crime-records 'month_index' has null values (corrupt parquet)");
|
||||||
|
}
|
||||||
|
match month_ca.min() {
|
||||||
|
Some(m) if m > 0 => {}
|
||||||
|
_ => bail!("crime-records 'month_index' must be a positive year*12+month index"),
|
||||||
|
}
|
||||||
|
let type_col = df
|
||||||
|
.column("crime_type")
|
||||||
|
.context("crime-records parquet missing 'crime_type'")?
|
||||||
|
.str()
|
||||||
|
.context("'crime_type' is not a string")?;
|
||||||
|
let location_col = df
|
||||||
|
.column("location")
|
||||||
|
.context("crime-records parquet missing 'location'")?
|
||||||
|
.str()
|
||||||
|
.context("'location' is not a string")?;
|
||||||
|
let outcome_col = df
|
||||||
|
.column("outcome")
|
||||||
|
.context("crime-records parquet missing 'outcome'")?
|
||||||
|
.str()
|
||||||
|
.context("'outcome' is not a string")?;
|
||||||
|
let lat_col = df
|
||||||
|
.column("lat")
|
||||||
|
.context("crime-records parquet missing 'lat'")?
|
||||||
|
.cast(&DataType::Float32)?;
|
||||||
|
let lat_ca = lat_col.f32().context("'lat' is not f32")?;
|
||||||
|
let lon_col = df
|
||||||
|
.column("lon")
|
||||||
|
.context("crime-records parquet missing 'lon'")?
|
||||||
|
.cast(&DataType::Float32)?;
|
||||||
|
let lon_ca = lon_col.f32().context("'lon' is not f32")?;
|
||||||
|
|
||||||
|
let height = df.height();
|
||||||
|
for row in 0..height {
|
||||||
|
// CSR index: the parquet is sorted by postcode, so a change in the
|
||||||
|
// postcode value (across chunk boundaries too) closes the previous
|
||||||
|
// run and opens a new one.
|
||||||
|
let pc = postcode_col
|
||||||
|
.get(row)
|
||||||
|
.with_context(|| {
|
||||||
|
format!("crime-records row {} has null postcode", offset + row)
|
||||||
|
})?
|
||||||
|
.trim();
|
||||||
|
if cur_pc.as_deref() != Some(pc) {
|
||||||
|
if let Some(prev) = cur_pc.take() {
|
||||||
|
by_postcode.insert(prev, (cur_start, global_row - cur_start));
|
||||||
|
}
|
||||||
|
cur_pc = Some(pc.to_string());
|
||||||
|
cur_start = global_row;
|
||||||
|
}
|
||||||
|
|
||||||
|
month.push(month_ca.get(row).unwrap() as u32);
|
||||||
|
|
||||||
|
let ty = type_col.get(row).unwrap_or("");
|
||||||
|
let ty_id = match type_index.get(ty) {
|
||||||
|
Some(&id) => id,
|
||||||
|
None => {
|
||||||
|
let id = u8::try_from(crime_type_dict.len())
|
||||||
|
.context("more than 256 distinct crime types")?;
|
||||||
|
crime_type_dict.push(ty.to_string());
|
||||||
|
type_index.insert(ty.to_string(), id);
|
||||||
|
id
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ctype.push(ty_id);
|
||||||
|
|
||||||
|
let oc = outcome_col.get(row).unwrap_or("");
|
||||||
|
let oc_id = if oc.is_empty() {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
match outcome_index.get(oc) {
|
||||||
|
Some(&id) => id,
|
||||||
|
None => {
|
||||||
|
let id = u8::try_from(outcome_dict.len())
|
||||||
|
.context("more than 256 distinct outcomes")?;
|
||||||
|
outcome_dict.push(oc.to_string());
|
||||||
|
outcome_index.insert(oc.to_string(), id);
|
||||||
|
id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
outcome.push(oc_id);
|
||||||
|
|
||||||
|
let loc = location_col.get(row).unwrap_or("");
|
||||||
|
location.push(if loc.is_empty() {
|
||||||
|
empty_spur
|
||||||
|
} else {
|
||||||
|
rodeo.get_or_intern(loc)
|
||||||
|
});
|
||||||
|
|
||||||
|
lat.push(lat_ca.get(row).unwrap_or(f32::NAN));
|
||||||
|
lon.push(lon_ca.get(row).unwrap_or(f32::NAN));
|
||||||
|
|
||||||
|
global_row += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += len;
|
||||||
|
}
|
||||||
|
if let Some(prev) = cur_pc.take() {
|
||||||
|
by_postcode.insert(prev, (cur_start, global_row - cur_start));
|
||||||
|
}
|
||||||
|
debug_assert_eq!(global_row as usize, n, "streamed fewer rows than the parquet declares");
|
||||||
|
|
||||||
|
let records = Self {
|
||||||
|
month: month.finish()?,
|
||||||
|
ctype: ctype.finish()?,
|
||||||
|
outcome: outcome.finish()?,
|
||||||
|
location: location.finish()?,
|
||||||
|
lat: lat.finish()?,
|
||||||
|
lon: lon.finish()?,
|
||||||
|
crime_type_dict,
|
||||||
|
outcome_dict,
|
||||||
|
location_resolver: rodeo.into_reader(),
|
||||||
|
by_postcode,
|
||||||
|
};
|
||||||
|
|
||||||
|
info!(
|
||||||
|
records = n,
|
||||||
|
postcodes = records.by_postcode.len(),
|
||||||
|
crime_types = records.crime_type_dict.len(),
|
||||||
|
outcomes = records.outcome_dict.len(),
|
||||||
|
"Crime records loaded"
|
||||||
|
);
|
||||||
|
Ok(records)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn write_fixture(path: &Path) {
|
||||||
|
// Two postcodes, postcode-sorted. AA1 1AA has 3 records across two
|
||||||
|
// months (a null outcome and a null location), BB2 2BB has 1.
|
||||||
|
let mut df = df!(
|
||||||
|
"postcode" => ["AA1 1AA", "AA1 1AA", "AA1 1AA", "BB2 2BB"],
|
||||||
|
"month_index" => [24300i32, 24300, 24290, 24305],
|
||||||
|
"crime_type" => ["Burglary", "Burglary", "Robbery", "Drugs"],
|
||||||
|
"location" => [Some("On or near A St"), Some("On or near A St"), None, Some("On or near B Rd")],
|
||||||
|
"outcome" => [Some("Under investigation"), None, Some("Court result"), None],
|
||||||
|
"lat" => [51.5f32, 51.5, 51.6, 52.0],
|
||||||
|
"lon" => [-0.1f32, -0.1, -0.2, -1.0],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut file = std::fs::File::create(path).unwrap();
|
||||||
|
ParquetWriter::new(&mut file).finish(&mut df).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn loads_indexes_and_gathers() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("crimerec-{}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let path = dir.join("records.parquet");
|
||||||
|
write_fixture(&path);
|
||||||
|
|
||||||
|
let recs = CrimeRecords::load(&path, None).unwrap();
|
||||||
|
assert_eq!(recs.total_for("AA1 1AA"), 3);
|
||||||
|
assert_eq!(recs.total_for("BB2 2BB"), 1);
|
||||||
|
assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
|
||||||
|
|
||||||
|
// Newest-first across the two postcodes.
|
||||||
|
let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
|
||||||
|
assert_eq!(all.len(), 4);
|
||||||
|
let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
|
||||||
|
assert_eq!(months, vec![24305, 24300, 24300, 24290]);
|
||||||
|
|
||||||
|
// `since` window filter (keep months >= 24300).
|
||||||
|
assert_eq!(recs.gather(&["AA1 1AA"], Some(24300)).len(), 2);
|
||||||
|
|
||||||
|
// String resolution + null handling.
|
||||||
|
let robbery = all
|
||||||
|
.iter()
|
||||||
|
.map(|&i| recs.view(i))
|
||||||
|
.find(|v| v.crime_type == "Robbery")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(robbery.outcome, Some("Court result"));
|
||||||
|
assert_eq!(robbery.location, None); // null location → None
|
||||||
|
// Two records have a null outcome (an AA1 Burglary and the BB2 Drugs).
|
||||||
|
let null_outcomes = all
|
||||||
|
.iter()
|
||||||
|
.map(|&i| recs.view(i))
|
||||||
|
.filter(|v| v.outcome.is_none())
|
||||||
|
.count();
|
||||||
|
assert_eq!(null_outcomes, 2);
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&dir).ok();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The CSR per-postcode index and the column builders must compose correctly
|
||||||
|
/// across streaming chunk boundaries — including a postcode run split between
|
||||||
|
/// two chunks. Forces `chunk_rows = 2` over the 4-row fixture so AA1 1AA's
|
||||||
|
/// three records straddle the boundary (rows 0,1 in chunk 0; row 2 in chunk 1)
|
||||||
|
/// and is exercised both heap-backed (no spill) and mmap-backed (spill).
|
||||||
|
#[test]
|
||||||
|
fn streams_across_chunk_boundaries() {
|
||||||
|
let base = std::env::temp_dir().join(format!("crimerec-chunk-{}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&base).unwrap();
|
||||||
|
let path = base.join("records.parquet");
|
||||||
|
write_fixture(&path);
|
||||||
|
let spill = base.join("spill");
|
||||||
|
std::fs::create_dir_all(&spill).unwrap();
|
||||||
|
|
||||||
|
for spill_dir in [None, Some(spill.as_path())] {
|
||||||
|
let recs = CrimeRecords::load_inner(&path, spill_dir, 2).unwrap();
|
||||||
|
// Counts match regardless of how the runs were split across chunks.
|
||||||
|
assert_eq!(recs.total_for("AA1 1AA"), 3);
|
||||||
|
assert_eq!(recs.total_for("BB2 2BB"), 1);
|
||||||
|
assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
|
||||||
|
|
||||||
|
// Full gather, newest-first, identical to the single-chunk load.
|
||||||
|
let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
|
||||||
|
assert_eq!(all.len(), 4);
|
||||||
|
let months: Vec<u32> = all.iter().map(|&i| recs.view(i).month_index).collect();
|
||||||
|
assert_eq!(months, vec![24305, 24300, 24300, 24290]);
|
||||||
|
|
||||||
|
// The run that straddled the boundary still resolves its strings.
|
||||||
|
let robbery = all
|
||||||
|
.iter()
|
||||||
|
.map(|&i| recs.view(i))
|
||||||
|
.find(|v| v.crime_type == "Robbery")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(robbery.outcome, Some("Court result"));
|
||||||
|
assert_eq!(robbery.location, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&base).ok();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Peak/resident RSS in MiB from `/proc/self/status` (Linux only).
|
||||||
|
fn rss_mib() -> (f64, f64) {
|
||||||
|
let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
|
||||||
|
let field = |key: &str| -> f64 {
|
||||||
|
status
|
||||||
|
.lines()
|
||||||
|
.find(|l| l.starts_with(key))
|
||||||
|
.and_then(|l| l.split_whitespace().nth(1))
|
||||||
|
.and_then(|kb| kb.parse::<f64>().ok())
|
||||||
|
.map_or(0.0, |kb| kb / 1024.0)
|
||||||
|
};
|
||||||
|
(field("VmHWM:"), field("VmRSS:"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Manual, real-data smoke test: load the actual ~500M-row parquet and report
|
||||||
|
/// peak RSS, proving the streaming + spill load completes without the
|
||||||
|
/// tens-of-GB `DataFrame` materialisation that OOMed the old `.collect()`.
|
||||||
|
///
|
||||||
|
/// Run with:
|
||||||
|
/// PPC_REAL_CRIME_RECORDS=/path/to/crime_records.parquet \
|
||||||
|
/// cargo test --bins -- --ignored --nocapture real_crime_records_load_is_bounded
|
||||||
|
#[test]
|
||||||
|
#[ignore = "needs the full crime_records.parquet; run manually"]
|
||||||
|
fn real_crime_records_load_is_bounded() {
|
||||||
|
let path = std::env::var("PPC_REAL_CRIME_RECORDS")
|
||||||
|
.unwrap_or_else(|_| "../property-data/crime_records.parquet".to_string());
|
||||||
|
let path = Path::new(&path);
|
||||||
|
if !path.exists() {
|
||||||
|
eprintln!("skipping: {} not found", path.display());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let spill = std::env::var("PPC_REAL_SPILL")
|
||||||
|
.unwrap_or_else(|_| "../.tmp/crime-spill-realtest".to_string());
|
||||||
|
let spill = Path::new(&spill);
|
||||||
|
std::fs::create_dir_all(spill).unwrap();
|
||||||
|
|
||||||
|
let (hwm_before, _rss_before) = rss_mib();
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let recs = CrimeRecords::load(path, Some(spill)).unwrap();
|
||||||
|
let elapsed = start.elapsed();
|
||||||
|
let (hwm_after, rss_after) = rss_mib();
|
||||||
|
|
||||||
|
let total: u64 = recs.by_postcode.values().map(|&(_, c)| c as u64).sum();
|
||||||
|
eprintln!(
|
||||||
|
"loaded {} records across {} postcodes in {:.1}s | RSS peak {:.0}->{:.0} MiB (Δ{:.0}) resident now {:.0} MiB",
|
||||||
|
total,
|
||||||
|
recs.by_postcode.len(),
|
||||||
|
elapsed.as_secs_f64(),
|
||||||
|
hwm_before,
|
||||||
|
hwm_after,
|
||||||
|
hwm_after - hwm_before,
|
||||||
|
rss_after,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(recs.by_postcode.len() > 0, "expected at least one postcode");
|
||||||
|
assert!(total > 0, "expected at least one record");
|
||||||
|
// The old `.collect()` decoded all rows' string columns at once (tens of
|
||||||
|
// GB). Streaming must keep the peak growth far below that; a generous 20GiB
|
||||||
|
// ceiling still proves we never materialise the whole file.
|
||||||
|
assert!(
|
||||||
|
hwm_after - hwm_before < 20_480.0,
|
||||||
|
"peak RSS grew by {:.0} MiB during load — streaming/spill not bounding memory",
|
||||||
|
hwm_after - hwm_before
|
||||||
|
);
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(spill).ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -26,9 +26,7 @@ use rustc_hash::FxHashMap;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::consts::NAN_U16;
|
use crate::consts::NAN_U16;
|
||||||
use crate::data::area_crime_averages::{AreaCrimeAverages, AVG_YR_SUFFIX};
|
|
||||||
use crate::data::spill::SpillVec;
|
use crate::data::spill::SpillVec;
|
||||||
use crate::utils::{postcode_outcode, postcode_sector};
|
|
||||||
|
|
||||||
#[derive(Serialize, Clone)]
|
#[derive(Serialize, Clone)]
|
||||||
pub struct RenovationEvent {
|
pub struct RenovationEvent {
|
||||||
|
|
@ -226,109 +224,6 @@ impl PropertyData {
|
||||||
num_numeric: self.num_numeric,
|
num_numeric: self.num_numeric,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Precompute mean headline crime rates nationally and per outcode / postcode
|
|
||||||
/// sector.
|
|
||||||
///
|
|
||||||
/// Crime values are identical for every property in a postcode (the pipeline
|
|
||||||
/// merges them on the postcode key), so each postcode is sampled once from
|
|
||||||
/// its first row and property-weighted by its row count. All three scopes use
|
|
||||||
/// the same exact property-weighted estimator over the same row universe as
|
|
||||||
/// the per-selection mean, so the four numbers shown in a crime row (this
|
|
||||||
/// selection / sector / outcode / nation) are directly comparable — without
|
|
||||||
/// the upward bias of the histogram-bin national average.
|
|
||||||
pub fn compute_area_crime_averages(&self) -> AreaCrimeAverages {
|
|
||||||
// Crime headline columns are exactly the " (avg/yr)" features.
|
|
||||||
let crime_indices: Vec<usize> = self
|
|
||||||
.feature_names
|
|
||||||
.iter()
|
|
||||||
.enumerate()
|
|
||||||
.filter(|(_, name)| name.ends_with(AVG_YR_SUFFIX))
|
|
||||||
.map(|(idx, _)| idx)
|
|
||||||
.collect();
|
|
||||||
if crime_indices.is_empty() {
|
|
||||||
return AreaCrimeAverages::empty();
|
|
||||||
}
|
|
||||||
let crime_types: Vec<String> = crime_indices
|
|
||||||
.iter()
|
|
||||||
.map(|&idx| {
|
|
||||||
self.feature_names[idx]
|
|
||||||
.strip_suffix(AVG_YR_SUFFIX)
|
|
||||||
.unwrap_or(&self.feature_names[idx])
|
|
||||||
.to_string()
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
let n = crime_indices.len();
|
|
||||||
|
|
||||||
// (weighted value sum, weight) accumulators per crime type.
|
|
||||||
let mut nat_sums = vec![0.0f64; n];
|
|
||||||
let mut nat_weights = vec![0u64; n];
|
|
||||||
let mut out_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
|
|
||||||
let mut sec_acc: FxHashMap<String, (Vec<f64>, Vec<u64>)> = FxHashMap::default();
|
|
||||||
|
|
||||||
for (key, rows) in &self.postcode_row_index {
|
|
||||||
let Some(&first) = rows.first() else { continue };
|
|
||||||
let count = rows.len() as u64;
|
|
||||||
let postcode = self.postcode_interner.resolve(key);
|
|
||||||
let outcode = postcode_outcode(postcode);
|
|
||||||
let sector = postcode_sector(postcode);
|
|
||||||
|
|
||||||
for (j, &fi) in crime_indices.iter().enumerate() {
|
|
||||||
// A NaN value is "no crime data for this postcode" — skip it so
|
|
||||||
// it dilutes neither the sum nor the weight (a genuine gap, not
|
|
||||||
// a zero), exactly as the global histogram excludes it.
|
|
||||||
let value = self.get_feature(first as usize, fi);
|
|
||||||
if !value.is_finite() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let weighted = value as f64 * count as f64;
|
|
||||||
// National counts every postcode (the population the global mean
|
|
||||||
// is built over); outcode/sector only when the postcode parses.
|
|
||||||
nat_sums[j] += weighted;
|
|
||||||
nat_weights[j] += count;
|
|
||||||
if let Some(outcode) = outcode {
|
|
||||||
let acc = out_acc
|
|
||||||
.entry(outcode.to_string())
|
|
||||||
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
|
|
||||||
acc.0[j] += weighted;
|
|
||||||
acc.1[j] += count;
|
|
||||||
}
|
|
||||||
if let Some(sector) = sector {
|
|
||||||
let acc = sec_acc
|
|
||||||
.entry(sector.to_string())
|
|
||||||
.or_insert_with(|| (vec![0.0; n], vec![0; n]));
|
|
||||||
acc.0[j] += weighted;
|
|
||||||
acc.1[j] += count;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let means_of = |sums: &[f64], weights: &[u64]| -> Vec<f32> {
|
|
||||||
sums.iter()
|
|
||||||
.zip(weights.iter())
|
|
||||||
.map(|(&sum, &weight)| {
|
|
||||||
if weight == 0 {
|
|
||||||
f32::NAN
|
|
||||||
} else {
|
|
||||||
(sum / weight as f64) as f32
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
};
|
|
||||||
let finalize =
|
|
||||||
|acc: FxHashMap<String, (Vec<f64>, Vec<u64>)>| -> FxHashMap<String, Vec<f32>> {
|
|
||||||
acc.into_iter()
|
|
||||||
.map(|(area, (sums, weights))| (area, means_of(&sums, &weights)))
|
|
||||||
.collect()
|
|
||||||
};
|
|
||||||
|
|
||||||
AreaCrimeAverages {
|
|
||||||
crime_types,
|
|
||||||
national: means_of(&nat_sums, &nat_weights),
|
|
||||||
by_outcode: finalize(out_acc),
|
|
||||||
by_sector: finalize(sec_acc),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -133,6 +133,138 @@ impl SpillVec<u16> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Builds a [`SpillVec<T>`] incrementally by `push`ing elements one at a time,
|
||||||
|
/// when the final length is known up front but the values arrive in a stream.
|
||||||
|
///
|
||||||
|
/// When a spill `dir` is set (and the length is non-zero) the backing store is a
|
||||||
|
/// pre-sized, memory-mapped file and each pushed element is written straight into
|
||||||
|
/// it — so the array never exists as a heap `Vec` and is never copied a second
|
||||||
|
/// time on finalisation, unlike [`SpillVec::maybe_spill`], which takes an
|
||||||
|
/// already-built `Vec` and so needs the whole thing resident first. Without a
|
||||||
|
/// spill dir it accumulates into an owned `Vec` (production behaviour, identical
|
||||||
|
/// resident cost to the old `Vec::with_capacity` + `maybe_spill`). This is what
|
||||||
|
/// lets the ~500M-row crime-records columns load without a multi-GB heap spike.
|
||||||
|
///
|
||||||
|
/// Exactly `len` elements must be pushed: pushing more panics, and finishing with
|
||||||
|
/// fewer is an error.
|
||||||
|
pub struct SpillVecBuilder<T: SpillElem> {
|
||||||
|
backing: Builder<T>,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Builder<T: SpillElem> {
|
||||||
|
Owned { values: Vec<T>, len: usize },
|
||||||
|
Mapped {
|
||||||
|
map: MmapMut,
|
||||||
|
len: usize,
|
||||||
|
cursor: usize,
|
||||||
|
label: &'static str,
|
||||||
|
_marker: PhantomData<T>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T: SpillElem> SpillVecBuilder<T> {
|
||||||
|
/// Create a builder for exactly `len` elements. Spills to `dir` when it is set
|
||||||
|
/// and `len > 0` (a zero-length mmap is invalid); otherwise reserves an owned
|
||||||
|
/// `Vec`. `label` names the backing file for diagnostics.
|
||||||
|
pub fn with_len(len: usize, dir: Option<&Path>, label: &'static str) -> anyhow::Result<Self> {
|
||||||
|
match dir {
|
||||||
|
Some(dir) if len > 0 => {
|
||||||
|
let byte_len = len * std::mem::size_of::<T>();
|
||||||
|
let file = anon_file(dir, label)?;
|
||||||
|
allocate_spill_file(&file, byte_len, label)?;
|
||||||
|
// SAFETY: `file` is a freshly-created, exclusively-owned regular
|
||||||
|
// file sized to exactly `byte_len`; no other mapping aliases it.
|
||||||
|
let map = unsafe { MmapMut::map_mut(&file) }.with_context(|| {
|
||||||
|
format!("mapping spill file for '{label}' ({byte_len} bytes)")
|
||||||
|
})?;
|
||||||
|
Ok(Self {
|
||||||
|
backing: Builder::Mapped {
|
||||||
|
map,
|
||||||
|
len,
|
||||||
|
cursor: 0,
|
||||||
|
label,
|
||||||
|
_marker: PhantomData,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
_ => Ok(Self {
|
||||||
|
backing: Builder::Owned {
|
||||||
|
values: Vec::with_capacity(len),
|
||||||
|
len,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append one element. Panics if more than the declared `len` elements are
|
||||||
|
/// pushed — enforced identically on both backings so a streaming bug fails
|
||||||
|
/// the same way in production (owned) as in dev (spilled).
|
||||||
|
#[inline]
|
||||||
|
pub fn push(&mut self, value: T) {
|
||||||
|
match &mut self.backing {
|
||||||
|
Builder::Owned { values, len } => {
|
||||||
|
assert!(
|
||||||
|
values.len() < *len,
|
||||||
|
"SpillVecBuilder overflow: pushed more than {len} elements"
|
||||||
|
);
|
||||||
|
values.push(value);
|
||||||
|
}
|
||||||
|
Builder::Mapped {
|
||||||
|
map, len, cursor, ..
|
||||||
|
} => {
|
||||||
|
assert!(
|
||||||
|
*cursor < *len,
|
||||||
|
"SpillVecBuilder overflow: pushed more than {len} elements"
|
||||||
|
);
|
||||||
|
// SAFETY: an mmap base is page-aligned (hence aligned for `T`); the
|
||||||
|
// mapping holds exactly `len * size_of::<T>()` bytes and `cursor <
|
||||||
|
// len`, so this writes one in-bounds, aligned `T`. `T: SpillElem`
|
||||||
|
// is `Copy` and padding-free, so the stored byte image is fully
|
||||||
|
// defined and reads back as the same value.
|
||||||
|
unsafe { map.as_mut_ptr().cast::<T>().add(*cursor).write(value) };
|
||||||
|
*cursor += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seal the builder into a read-only [`SpillVec`]. Exactly the declared `len`
|
||||||
|
/// elements must have been pushed, on both backings.
|
||||||
|
pub fn finish(self) -> anyhow::Result<SpillVec<T>> {
|
||||||
|
match self.backing {
|
||||||
|
Builder::Owned { values, len } => {
|
||||||
|
if values.len() != len {
|
||||||
|
anyhow::bail!(
|
||||||
|
"spill builder finished with {} of {len} elements written",
|
||||||
|
values.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(SpillVec::Owned(values))
|
||||||
|
}
|
||||||
|
Builder::Mapped {
|
||||||
|
map,
|
||||||
|
len,
|
||||||
|
cursor,
|
||||||
|
label,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
if cursor != len {
|
||||||
|
anyhow::bail!(
|
||||||
|
"spill builder for '{label}' finished with {cursor} of {len} elements written"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let map = map
|
||||||
|
.make_read_only()
|
||||||
|
.with_context(|| format!("sealing spill file for '{label}'"))?;
|
||||||
|
Ok(SpillVec::Mapped {
|
||||||
|
map,
|
||||||
|
len,
|
||||||
|
_marker: PhantomData,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<T: SpillElem> std::ops::Deref for SpillVec<T> {
|
impl<T: SpillElem> std::ops::Deref for SpillVec<T> {
|
||||||
type Target = [T];
|
type Target = [T];
|
||||||
|
|
||||||
|
|
@ -327,4 +459,105 @@ mod tests {
|
||||||
assert!(matches!(mapped, SpillVec::Owned(_)));
|
assert!(matches!(mapped, SpillVec::Owned(_)));
|
||||||
assert!(mapped.is_empty());
|
assert!(mapped.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builder_streams_identically_owned_and_mapped() {
|
||||||
|
let values: Vec<u32> = (0..50_000u32).map(|n| n.wrapping_mul(2_246_822_519)).collect();
|
||||||
|
|
||||||
|
// Owned path (no spill dir): pushes accumulate into a heap Vec.
|
||||||
|
let mut owned = SpillVecBuilder::<u32>::with_len(values.len(), None, "u32_owned").unwrap();
|
||||||
|
for &v in &values {
|
||||||
|
owned.push(v);
|
||||||
|
}
|
||||||
|
let owned = owned.finish().unwrap();
|
||||||
|
assert!(matches!(owned, SpillVec::Owned(_)));
|
||||||
|
assert_eq!(&*owned, values.as_slice());
|
||||||
|
|
||||||
|
// Mapped path (spill dir): pushes write straight into the mmap, no heap copy.
|
||||||
|
let dir = TempDir::new("builder-u32");
|
||||||
|
let mut mapped =
|
||||||
|
SpillVecBuilder::<u32>::with_len(values.len(), Some(dir.path()), "u32_mapped").unwrap();
|
||||||
|
for &v in &values {
|
||||||
|
mapped.push(v);
|
||||||
|
}
|
||||||
|
let mapped = mapped.finish().unwrap();
|
||||||
|
assert!(matches!(mapped, SpillVec::Mapped { .. }));
|
||||||
|
// The mmap-backed slice must be byte-identical to the streamed input.
|
||||||
|
assert_eq!(&*mapped, values.as_slice());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builder_spurs_survive_the_mmap_roundtrip() {
|
||||||
|
// Spur is a NonZeroU32 niche type — exercises the streamed write path for
|
||||||
|
// the crime-records `location` column.
|
||||||
|
let mut rodeo = lasso::Rodeo::default();
|
||||||
|
let keys: Vec<lasso::Spur> = (0..3000)
|
||||||
|
.map(|n| rodeo.get_or_intern(format!("loc-{n}")))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let dir = TempDir::new("builder-spur");
|
||||||
|
let mut builder =
|
||||||
|
SpillVecBuilder::<lasso::Spur>::with_len(keys.len(), Some(dir.path()), "spurs").unwrap();
|
||||||
|
for &k in &keys {
|
||||||
|
builder.push(k);
|
||||||
|
}
|
||||||
|
let mapped = builder.finish().unwrap();
|
||||||
|
assert!(matches!(mapped, SpillVec::Mapped { .. }));
|
||||||
|
assert_eq!(&*mapped, keys.as_slice());
|
||||||
|
let reader = rodeo.into_resolver();
|
||||||
|
for (idx, &key) in mapped.iter().enumerate() {
|
||||||
|
assert_eq!(reader.resolve(&key), format!("loc-{idx}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builder_zero_len_stays_owned() {
|
||||||
|
let dir = TempDir::new("builder-empty");
|
||||||
|
let builder = SpillVecBuilder::<u32>::with_len(0, Some(dir.path()), "empty").unwrap();
|
||||||
|
let v = builder.finish().unwrap();
|
||||||
|
// A zero-length mmap is invalid, so empties fall back to an owned Vec.
|
||||||
|
assert!(matches!(v, SpillVec::Owned(_)));
|
||||||
|
assert!(v.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builder_underfill_is_an_error() {
|
||||||
|
let dir = TempDir::new("builder-underfill");
|
||||||
|
let mut builder =
|
||||||
|
SpillVecBuilder::<u32>::with_len(4, Some(dir.path()), "underfill").unwrap();
|
||||||
|
builder.push(1);
|
||||||
|
builder.push(2);
|
||||||
|
// Sealing a spilling builder before all declared elements are written fails
|
||||||
|
// rather than exposing uninitialised mmap tail bytes as valid data.
|
||||||
|
assert!(builder.finish().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[should_panic(expected = "overflow")]
|
||||||
|
fn builder_overfill_panics() {
|
||||||
|
let dir = TempDir::new("builder-overfill");
|
||||||
|
let mut builder = SpillVecBuilder::<u32>::with_len(2, Some(dir.path()), "overfill").unwrap();
|
||||||
|
builder.push(1);
|
||||||
|
builder.push(2);
|
||||||
|
builder.push(3); // one past the declared length
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builder_owned_underfill_is_an_error() {
|
||||||
|
// The owned (no-spill, production) path enforces the declared length too,
|
||||||
|
// so a streaming bug can't silently yield a short array in release builds.
|
||||||
|
let mut builder = SpillVecBuilder::<u32>::with_len(4, None, "owned-underfill").unwrap();
|
||||||
|
builder.push(1);
|
||||||
|
builder.push(2);
|
||||||
|
assert!(builder.finish().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[should_panic(expected = "overflow")]
|
||||||
|
fn builder_owned_overfill_panics() {
|
||||||
|
let mut builder = SpillVecBuilder::<u32>::with_len(2, None, "owned-overfill").unwrap();
|
||||||
|
builder.push(1);
|
||||||
|
builder.push(2);
|
||||||
|
builder.push(3); // one past the declared length
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,59 @@ pub struct FeatureGroup {
|
||||||
pub features: &'static [Feature],
|
pub features: &'static [Feature],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Expand each crime type into its two filterable features: a 7-year and a
|
||||||
|
/// 2-year window. Each is the average number of recorded incidents per year (the
|
||||||
|
/// raw, absolute count — no per-area or per-capita normalisation). The names must
|
||||||
|
/// match the `"{type} (/yr, 7y|2y)"` columns written by `crime_spatial`. The
|
||||||
|
/// per-incident records are NOT a feature (they are a display-only side table the
|
||||||
|
/// server loads directly), so they never appear here and are not filterable.
|
||||||
|
macro_rules! crime_features {
|
||||||
|
($( ($base:literal, $blurb:literal) ),+ $(,)?) => {
|
||||||
|
&[ $(
|
||||||
|
Feature::Numeric(FeatureConfig {
|
||||||
|
name: concat!($base, " (/yr, 7y)"),
|
||||||
|
bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
|
||||||
|
step: 0.1,
|
||||||
|
description: concat!($blurb, " — average recorded incidents per year (last 7 years)"),
|
||||||
|
detail: concat!(
|
||||||
|
$blurb,
|
||||||
|
", as the average number of recorded incidents per year, over the last \
|
||||||
|
7 years. Counted from police.uk street-level crime points (anonymised, \
|
||||||
|
snapped to nearby map points) that fall near the postcode boundary — \
|
||||||
|
the raw, absolute count, with no per-area or per-capita adjustment. \
|
||||||
|
Computed over the months the local police force actually published; \
|
||||||
|
known force gaps (e.g. Greater Manchester since mid-2019) are excluded, \
|
||||||
|
not counted as zero crime."
|
||||||
|
),
|
||||||
|
source: "crime",
|
||||||
|
prefix: "",
|
||||||
|
suffix: "",
|
||||||
|
raw: false,
|
||||||
|
absolute: false,
|
||||||
|
}),
|
||||||
|
Feature::Numeric(FeatureConfig {
|
||||||
|
name: concat!($base, " (/yr, 2y)"),
|
||||||
|
bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
|
||||||
|
step: 0.1,
|
||||||
|
description: concat!($blurb, " — average recorded incidents per year (last 2 years)"),
|
||||||
|
detail: concat!(
|
||||||
|
$blurb,
|
||||||
|
", as the average number of recorded incidents per year, over the last \
|
||||||
|
2 years — a more recent but noisier window than the 7-year figure. From \
|
||||||
|
police.uk street-level crime points near the postcode boundary (the raw, \
|
||||||
|
absolute count), over the months the local force published (gaps \
|
||||||
|
excluded, not zeroed)."
|
||||||
|
),
|
||||||
|
source: "crime",
|
||||||
|
prefix: "",
|
||||||
|
suffix: "",
|
||||||
|
raw: false,
|
||||||
|
absolute: false,
|
||||||
|
}),
|
||||||
|
)+ ]
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
||||||
FeatureGroup {
|
FeatureGroup {
|
||||||
name: "Properties",
|
name: "Properties",
|
||||||
|
|
@ -472,247 +525,41 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
||||||
},
|
},
|
||||||
FeatureGroup {
|
FeatureGroup {
|
||||||
name: "Crime",
|
name: "Crime",
|
||||||
features: &[
|
features: crime_features![
|
||||||
Feature::Numeric(FeatureConfig {
|
(
|
||||||
name: "Serious crime (avg/yr)",
|
"Serious crime",
|
||||||
bounds: Bounds::Percentile {
|
"Serious crime — violence, robbery, burglary and weapons possession — near the postcode"
|
||||||
low: 2.0,
|
),
|
||||||
high: 98.0,
|
(
|
||||||
},
|
"Minor crime",
|
||||||
step: 1.0,
|
"Lower-severity crime — anti-social behaviour, theft, criminal damage, drugs and public order — near the postcode"
|
||||||
description: "Relative density of serious crime categories near the postcode",
|
),
|
||||||
detail: "Combined density of violence, robbery, burglary, and weapons possession near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
|
(
|
||||||
source: "crime",
|
"Violence and sexual offences",
|
||||||
prefix: "",
|
"Violence and sexual offences (assault, harassment, sexual offences) near the postcode"
|
||||||
suffix: "",
|
),
|
||||||
raw: false,
|
("Burglary", "Burglary (residential and commercial) near the postcode"),
|
||||||
absolute: false,
|
("Robbery", "Robbery (theft with force or threat) near the postcode"),
|
||||||
}),
|
("Vehicle crime", "Vehicle crime (theft of and from vehicles) near the postcode"),
|
||||||
Feature::Numeric(FeatureConfig {
|
("Anti-social behaviour", "Anti-social behaviour near the postcode"),
|
||||||
name: "Minor crime (avg/yr)",
|
("Criminal damage and arson", "Criminal damage and arson near the postcode"),
|
||||||
bounds: Bounds::Percentile {
|
(
|
||||||
low: 2.0,
|
"Other theft",
|
||||||
high: 98.0,
|
"Other theft (not burglary, vehicle, shoplifting or bicycle theft) near the postcode"
|
||||||
},
|
),
|
||||||
step: 1.0,
|
(
|
||||||
description: "Relative density of minor crime categories near the postcode",
|
"Theft from the person",
|
||||||
detail: "Combined density of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
|
"Theft from the person (pickpocketing, bag snatching) near the postcode"
|
||||||
source: "crime",
|
),
|
||||||
prefix: "",
|
("Shoplifting", "Shoplifting near the postcode"),
|
||||||
suffix: "",
|
("Bicycle theft", "Bicycle theft near the postcode"),
|
||||||
raw: false,
|
("Drugs", "Drug offences (possession and trafficking) near the postcode"),
|
||||||
absolute: false,
|
("Possession of weapons", "Possession of weapons near the postcode"),
|
||||||
}),
|
(
|
||||||
Feature::Numeric(FeatureConfig {
|
"Public order",
|
||||||
name: "Violence and sexual offences (avg/yr)",
|
"Public order offences (causing fear, alarm or distress) near the postcode"
|
||||||
bounds: Bounds::Percentile {
|
),
|
||||||
low: 2.0,
|
("Other crime", "Other crime (offences not classified elsewhere) near the postcode"),
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly violent and sexual offences in the area",
|
|
||||||
detail: "Average number of violence and sexual offences per year near the postcode, from police.uk street-level crime data. Includes assault, harassment, and sexual offences.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Burglary (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly burglary offences in the area",
|
|
||||||
detail: "Average number of burglary offences per year near the postcode, from police.uk street-level crime data. Includes residential and commercial burglary.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Robbery (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly robbery offences in the area",
|
|
||||||
detail: "Average number of robbery offences per year near the postcode, from police.uk street-level crime data. Robbery involves theft with force or threat of force.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Vehicle crime (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly vehicle crime in the area",
|
|
||||||
detail: "Average number of vehicle crime incidents per year near the postcode, from police.uk street-level crime data. Includes theft of and from vehicles.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Anti-social behaviour (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly anti-social behaviour incidents in the area",
|
|
||||||
detail: "Average number of anti-social behaviour incidents per year near the postcode, from police.uk street-level crime data. Includes nuisance, environmental, and personal anti-social behaviour.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Criminal damage and arson (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly criminal damage and arson in the area",
|
|
||||||
detail: "Average number of criminal damage and arson incidents per year near the postcode, from police.uk street-level crime data.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Other theft (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly other theft offences in the area",
|
|
||||||
detail: "Average number of 'other theft' offences per year near the postcode, from police.uk street-level crime data. Includes theft not classified under burglary, vehicle crime, shoplifting, or bicycle theft.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Theft from the person (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly theft from the person in the area",
|
|
||||||
detail: "Average number of theft from the person offences per year near the postcode, from police.uk street-level crime data. Includes pickpocketing and bag snatching without force.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Shoplifting (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly shoplifting offences in the area",
|
|
||||||
detail: "Average number of shoplifting offences per year near the postcode, from police.uk street-level crime data.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Bicycle theft (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly bicycle theft in the area",
|
|
||||||
detail: "Average number of bicycle theft offences per year near the postcode, from police.uk street-level crime data.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Drugs (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly drug offences in the area",
|
|
||||||
detail: "Average number of drug offences per year near the postcode, from police.uk street-level crime data. Includes possession and trafficking offences.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Possession of weapons (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly weapons possession offences in the area",
|
|
||||||
detail: "Average number of possession of weapons offences per year near the postcode, from police.uk street-level crime data.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Public order (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly public order offences in the area",
|
|
||||||
detail: "Average number of public order offences per year near the postcode, from police.uk street-level crime data. Includes causing fear, alarm, or distress.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Other crime (avg/yr)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 1.0,
|
|
||||||
description: "Average yearly other crime in the area",
|
|
||||||
detail: "Average number of other crime offences per year near the postcode, from police.uk street-level crime data. A catch-all category for offences not classified elsewhere.",
|
|
||||||
source: "crime",
|
|
||||||
prefix: "",
|
|
||||||
suffix: "",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
FeatureGroup {
|
FeatureGroup {
|
||||||
|
|
@ -826,9 +673,10 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
||||||
// shares sum to ~100% per neighbourhood (LSOA) and render as a
|
// shares sum to ~100% per neighbourhood (LSOA) and render as a
|
||||||
// stacked composition (see STACKED_GROUPS["Neighbours"] in the
|
// stacked composition (see STACKED_GROUPS["Neighbours"] in the
|
||||||
// frontend), like the ethnicity, qualifications and vote-share bars.
|
// frontend), like the ethnicity, qualifications and vote-share bars.
|
||||||
// Unlike those, the three shares are ALSO offered as individual
|
// Unlike those — each folded into a single dropdown filter that
|
||||||
// filters (they are not added to the display-only skip-list in
|
// selects one band — the three tenure shares are offered as
|
||||||
// Filters.tsx), so users can target e.g. owner-occupier-heavy areas.
|
// individual filters, so users can target e.g. owner-occupier-heavy
|
||||||
|
// areas.
|
||||||
Feature::Numeric(FeatureConfig {
|
Feature::Numeric(FeatureConfig {
|
||||||
name: "% Owner occupied",
|
name: "% Owner occupied",
|
||||||
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
|
||||||
|
|
@ -1292,7 +1140,7 @@ mod tests {
|
||||||
"Income Score", // 0..100 percentile
|
"Income Score", // 0..100 percentile
|
||||||
"% White", // 0..100 percentage
|
"% White", // 0..100 percentage
|
||||||
"Noise (dB)", // 50..80, range > threshold
|
"Noise (dB)", // 50..80, range > threshold
|
||||||
"Serious crime (avg/yr)", // Percentile bounds, fractional
|
"Serious crime (/yr, 7y)", // Percentile bounds, fractional
|
||||||
"Interior height (m)", // step 0.1
|
"Interior height (m)", // step 0.1
|
||||||
"Estimated current price", // step 10000
|
"Estimated current price", // step 10000
|
||||||
] {
|
] {
|
||||||
|
|
|
||||||
|
|
@ -334,6 +334,18 @@ struct Cli {
|
||||||
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
|
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
|
||||||
crime_by_year_path: PathBuf,
|
crime_by_year_path: PathBuf,
|
||||||
|
|
||||||
|
/// Path to the per-incident crime-records parquet (last 7 years, postcode-
|
||||||
|
/// sorted) backing the "individual crimes" list. Spilled to disk when
|
||||||
|
/// `--spill-dir` is set.
|
||||||
|
#[arg(long, env = "CRIME_RECORDS_PATH")]
|
||||||
|
crime_records_path: PathBuf,
|
||||||
|
|
||||||
|
/// Path to the precomputed national/per-outcode/per-sector crime-averages
|
||||||
|
/// parquet (built by pipeline.transform.area_crime_averages). The right pane
|
||||||
|
/// uses it to compare a selection's crime rates against its surroundings.
|
||||||
|
#[arg(long, env = "AREA_CRIME_AVERAGES_PATH")]
|
||||||
|
area_crime_averages_path: PathBuf,
|
||||||
|
|
||||||
/// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
|
/// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
|
||||||
/// residents; display-only side table for the right pane). Optional: when
|
/// residents; display-only side table for the right pane). Optional: when
|
||||||
/// absent or missing, the area pane simply omits the population figure.
|
/// absent or missing, the area pane simply omits the population figure.
|
||||||
|
|
@ -725,6 +737,16 @@ async fn main() -> anyhow::Result<()> {
|
||||||
Arc::new(data)
|
Arc::new(data)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let crime_records = {
|
||||||
|
let path = &cli.crime_records_path;
|
||||||
|
if !path.exists() {
|
||||||
|
bail!("Crime-records parquet not found: {}", path.display());
|
||||||
|
}
|
||||||
|
let data = data::CrimeRecords::load(path, spill_dir)?;
|
||||||
|
trim_allocator("crime-records load");
|
||||||
|
Arc::new(data)
|
||||||
|
};
|
||||||
|
|
||||||
let population = match &cli.population_path {
|
let population = match &cli.population_path {
|
||||||
Some(path) if path.exists() => {
|
Some(path) if path.exists() => {
|
||||||
let data = data::PostcodePopulation::load(path)?;
|
let data = data::PostcodePopulation::load(path)?;
|
||||||
|
|
@ -742,13 +764,11 @@ async fn main() -> anyhow::Result<()> {
|
||||||
};
|
};
|
||||||
|
|
||||||
let area_crime_averages = {
|
let area_crime_averages = {
|
||||||
let data = property_data.compute_area_crime_averages();
|
let path = &cli.area_crime_averages_path;
|
||||||
info!(
|
if !path.exists() {
|
||||||
outcodes = data.by_outcode.len(),
|
bail!("Area-crime-averages parquet not found: {}", path.display());
|
||||||
sectors = data.by_sector.len(),
|
}
|
||||||
crime_types = data.crime_types.len(),
|
let data = data::AreaCrimeAverages::load(path)?;
|
||||||
"Per-outcode/sector crime averages computed"
|
|
||||||
);
|
|
||||||
trim_allocator("area crime averages");
|
trim_allocator("area crime averages");
|
||||||
Arc::new(data)
|
Arc::new(data)
|
||||||
};
|
};
|
||||||
|
|
@ -781,6 +801,7 @@ async fn main() -> anyhow::Result<()> {
|
||||||
actual_listings,
|
actual_listings,
|
||||||
developments,
|
developments,
|
||||||
crime_by_year,
|
crime_by_year,
|
||||||
|
crime_records,
|
||||||
population,
|
population,
|
||||||
area_crime_averages,
|
area_crime_averages,
|
||||||
token_cache,
|
token_cache,
|
||||||
|
|
@ -899,6 +920,10 @@ async fn main() -> anyhow::Result<()> {
|
||||||
"/api/postcode-properties",
|
"/api/postcode-properties",
|
||||||
get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
|
get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/api/crime-records",
|
||||||
|
get(routes::get_crime_records).layer(ConcurrencyLimitLayer::new(5)),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/screenshot",
|
"/api/screenshot",
|
||||||
get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),
|
get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),
|
||||||
|
|
|
||||||
|
|
@ -1319,6 +1319,49 @@ pub async fn ensure_collections(
|
||||||
ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?;
|
ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Per-user record of which property listings a user has opened, so visited listings
|
||||||
|
// can be drawn in a distinct colour on the map. One row per (user, url); the unique
|
||||||
|
// index makes re-clicks idempotent.
|
||||||
|
let clicked_listings_index =
|
||||||
|
"CREATE UNIQUE INDEX idx_clicked_listings_user_url ON clicked_listings (user, url)";
|
||||||
|
if !existing.iter().any(|n| n == "clicked_listings") {
|
||||||
|
let users_id = find_users_collection_id(client, base_url, &token).await?;
|
||||||
|
let user_only = Some("user = @request.auth.id".to_string());
|
||||||
|
create_collection(
|
||||||
|
client,
|
||||||
|
base_url,
|
||||||
|
&token,
|
||||||
|
CreateCollection {
|
||||||
|
name: "clicked_listings".to_string(),
|
||||||
|
r#type: "base".to_string(),
|
||||||
|
fields: vec![
|
||||||
|
Field::relation("user", &users_id),
|
||||||
|
Field::text("url", true),
|
||||||
|
Field::autodate("created", true, false),
|
||||||
|
Field::autodate("updated", true, true),
|
||||||
|
],
|
||||||
|
list_rule: user_only.clone(),
|
||||||
|
view_rule: user_only.clone(),
|
||||||
|
create_rule: user_only.clone(),
|
||||||
|
update_rule: user_only.clone(),
|
||||||
|
delete_rule: user_only,
|
||||||
|
indexes: vec![clicked_listings_index.to_string()],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
} else {
|
||||||
|
ensure_user_owned_rules(client, base_url, &token, "clicked_listings").await?;
|
||||||
|
ensure_autodate_fields(client, base_url, &token, "clicked_listings").await?;
|
||||||
|
ensure_collection_indexes(
|
||||||
|
client,
|
||||||
|
base_url,
|
||||||
|
&token,
|
||||||
|
"clicked_listings",
|
||||||
|
&[("idx_clicked_listings_user_url", clicked_listings_index)],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
mod actual_listings;
|
mod actual_listings;
|
||||||
mod ai_filters;
|
mod ai_filters;
|
||||||
mod checkout;
|
mod checkout;
|
||||||
|
mod crime_records;
|
||||||
mod developments;
|
mod developments;
|
||||||
mod export;
|
mod export;
|
||||||
mod features;
|
mod features;
|
||||||
|
|
@ -35,6 +36,7 @@ pub(crate) mod travel_time;
|
||||||
pub use actual_listings::get_actual_listings;
|
pub use actual_listings::get_actual_listings;
|
||||||
pub use ai_filters::{build_system_prompt, post_ai_filters};
|
pub use ai_filters::{build_system_prompt, post_ai_filters};
|
||||||
pub use checkout::post_checkout;
|
pub use checkout::post_checkout;
|
||||||
|
pub use crime_records::get_crime_records;
|
||||||
pub use developments::get_developments;
|
pub use developments::get_developments;
|
||||||
pub use export::get_export;
|
pub use export::get_export;
|
||||||
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
|
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
|
||||||
|
|
|
||||||
|
|
@ -336,8 +336,8 @@ mod tests {
|
||||||
"Good+ primary school catchments"
|
"Good+ primary school catchments"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
canonical_filter_name("Specific crimes:Burglary%20%28avg%2Fyr%29:1"),
|
canonical_filter_name("Specific crimes:Burglary%20%28%2Fyr%2C%207y%29:1"),
|
||||||
"Burglary (avg/yr)"
|
"Burglary (/yr, 7y)"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
canonical_filter_name("Political vote share:%25%20Labour:0"),
|
canonical_filter_name("Political vote share:%25%20Labour:0"),
|
||||||
|
|
|
||||||
|
|
@ -25,9 +25,9 @@ pub fn build_system_prompt(
|
||||||
or \"max\" (at most this value). Never set two filters on the same feature.\n\
|
or \"max\" (at most this value). Never set two filters on the same feature.\n\
|
||||||
- Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
|
- Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
|
||||||
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
|
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
|
||||||
- \"low crime\" / \"safe\" = low values on the Serious crime (avg/yr) and Minor crime (avg/yr) \
|
- \"low crime\" / \"safe\" = low values on the Serious crime (/yr, 7y) and Minor crime (/yr, 7y) \
|
||||||
features (area-normalised incident density near the postcode). Prefer these aggregates for broad \
|
features (average recorded incidents per year near the postcode, last 7 years). Prefer these aggregates for broad \
|
||||||
area safety; use specific crime features only when the user names a crime type.\n\
|
area safety; use specific crime features only when the user names a crime type. Use a \"(/yr, 2y)\" feature only when the user asks about recent crime.\n\
|
||||||
- \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \
|
- \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \
|
||||||
or low Distance to nearest park (km), depending on wording.\n\
|
or low Distance to nearest park (km), depending on wording.\n\
|
||||||
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
|
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
|
||||||
|
|
@ -171,8 +171,8 @@ pub fn build_system_prompt(
|
||||||
parts.push(
|
parts.push(
|
||||||
"\nUser: \"safe quiet area with good schools and parks\"\n\
|
"\nUser: \"safe quiet area with good schools and parks\"\n\
|
||||||
Output: {\"numeric_filters\": [\
|
Output: {\"numeric_filters\": [\
|
||||||
{\"name\": \"Serious crime (avg/yr)\", \"bound\": \"max\", \"value\": 5}, \
|
{\"name\": \"Serious crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 5}, \
|
||||||
{\"name\": \"Minor crime (avg/yr)\", \"bound\": \"max\", \"value\": 20}, \
|
{\"name\": \"Minor crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 20}, \
|
||||||
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
|
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
|
||||||
{\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \
|
{\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \
|
||||||
{\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \
|
{\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \
|
||||||
|
|
@ -237,7 +237,7 @@ pub fn build_system_prompt(
|
||||||
"\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\
|
"\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\
|
||||||
Output: {\"numeric_filters\": [\
|
Output: {\"numeric_filters\": [\
|
||||||
{\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \
|
{\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \
|
||||||
{\"name\": \"Burglary (avg/yr)\", \"bound\": \"max\", \"value\": 10}, \
|
{\"name\": \"Burglary (/yr, 7y)\", \"bound\": \"max\", \"value\": 10}, \
|
||||||
{\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \
|
{\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \
|
||||||
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
|
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
|
||||||
.to_string(),
|
.to_string(),
|
||||||
|
|
|
||||||
194
server-rs/src/routes/crime_records.rs
Normal file
194
server-rs/src/routes/crime_records.rs
Normal file
|
|
@ -0,0 +1,194 @@
|
||||||
|
//! `GET /api/crime-records` — the individual police.uk crimes (last 7 years)
|
||||||
|
//! behind a selected hexagon or postcode, paginated. Display-only and
|
||||||
|
//! independent of the property filters, like the population figure: the records
|
||||||
|
//! are an attribute of the area, not of the filter-matching subset.
|
||||||
|
|
||||||
|
use std::str::FromStr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use axum::extract::{Query, State};
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::response::{IntoResponse, Json};
|
||||||
|
use axum::Extension;
|
||||||
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tracing::{info, warn};
|
||||||
|
|
||||||
|
use crate::auth::OptionalUser;
|
||||||
|
use crate::licensing::{check_license_bounds, check_license_point, resolve_share_code};
|
||||||
|
use crate::parsing::{cell_for_row_cached, h3_cell_bounds, needs_parent, validate_h3_resolution};
|
||||||
|
use crate::state::SharedState;
|
||||||
|
use crate::utils::normalize_postcode;
|
||||||
|
|
||||||
|
/// Default and hard-cap page sizes for the records list.
|
||||||
|
const DEFAULT_LIMIT: usize = 200;
|
||||||
|
const MAX_LIMIT: usize = 500;
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct CrimeRecordsParams {
|
||||||
|
/// Hexagon selection: H3 cell + resolution. Mutually exclusive with `postcode`.
|
||||||
|
pub h3: Option<String>,
|
||||||
|
pub resolution: Option<u8>,
|
||||||
|
/// Postcode selection.
|
||||||
|
pub postcode: Option<String>,
|
||||||
|
pub offset: Option<usize>,
|
||||||
|
pub limit: Option<usize>,
|
||||||
|
/// Lower bound on `month_index` (`year*12 + month0`) to restrict to a recent
|
||||||
|
/// window; omitted = all stored records (last 7 years).
|
||||||
|
pub since: Option<u32>,
|
||||||
|
/// Share-link code; grants scoped access for unlicensed users.
|
||||||
|
pub share: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct CrimeRecord {
|
||||||
|
/// `"YYYY-MM"`.
|
||||||
|
pub month: String,
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
pub crime_type: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub location: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub outcome: Option<String>,
|
||||||
|
pub lat: f32,
|
||||||
|
pub lon: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct CrimeRecordsResponse {
|
||||||
|
pub records: Vec<CrimeRecord>,
|
||||||
|
pub total: usize,
|
||||||
|
pub offset: usize,
|
||||||
|
pub truncated: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_month(month_index: u32) -> String {
|
||||||
|
let year = month_index / 12;
|
||||||
|
let month = month_index % 12 + 1;
|
||||||
|
format!("{year:04}-{month:02}")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_crime_records(
|
||||||
|
State(shared): State<Arc<SharedState>>,
|
||||||
|
Extension(user): Extension<OptionalUser>,
|
||||||
|
Extension(geo): Extension<crate::demo_zone::DemoZone>,
|
||||||
|
Query(params): Query<CrimeRecordsParams>,
|
||||||
|
) -> Result<Json<CrimeRecordsResponse>, axum::response::Response> {
|
||||||
|
let state = shared.load_state();
|
||||||
|
let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
|
||||||
|
let offset = params.offset.unwrap_or(0);
|
||||||
|
let limit = params.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
|
||||||
|
let since = params.since;
|
||||||
|
|
||||||
|
// Resolve the selection to a set of postcodes, after a license check scoped
|
||||||
|
// to the selection's geometry (bounds for a hexagon, point for a postcode).
|
||||||
|
enum Selection {
|
||||||
|
Hexagon { cell: u64, resolution: u8 },
|
||||||
|
Postcode(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
let selection = if let Some(h3) = params.h3.clone() {
|
||||||
|
let cell = h3o::CellIndex::from_str(&h3).map_err(|error| {
|
||||||
|
warn!(h3 = %h3, error = %error, "Invalid H3 cell index");
|
||||||
|
(StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {error}")).into_response()
|
||||||
|
})?;
|
||||||
|
let resolution = params.resolution.ok_or_else(|| {
|
||||||
|
(StatusCode::BAD_REQUEST, "resolution is required with h3").into_response()
|
||||||
|
})?;
|
||||||
|
validate_h3_resolution(resolution).map_err(IntoResponse::into_response)?;
|
||||||
|
let bounds = h3_cell_bounds(cell, 0.0);
|
||||||
|
check_license_bounds(&user.0, bounds, geo.free_zone, share_bounds)?;
|
||||||
|
Selection::Hexagon {
|
||||||
|
cell: cell.into(),
|
||||||
|
resolution,
|
||||||
|
}
|
||||||
|
} else if let Some(postcode) = params.postcode.clone() {
|
||||||
|
let normalized = normalize_postcode(&postcode);
|
||||||
|
let &pc_idx = state
|
||||||
|
.postcode_data
|
||||||
|
.postcode_to_idx
|
||||||
|
.get(&normalized)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
(StatusCode::NOT_FOUND, format!("Postcode not found: {normalized}")).into_response()
|
||||||
|
})?;
|
||||||
|
let (lat, lon) = state.postcode_data.centroids[pc_idx];
|
||||||
|
check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
|
||||||
|
Selection::Postcode(normalized)
|
||||||
|
} else {
|
||||||
|
return Err((StatusCode::BAD_REQUEST, "h3 or postcode is required").into_response());
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = tokio::task::spawn_blocking(move || -> Result<CrimeRecordsResponse, String> {
|
||||||
|
// Distinct postcodes covered by the selection.
|
||||||
|
let postcodes: Vec<String> = match selection {
|
||||||
|
Selection::Postcode(pc) => vec![pc],
|
||||||
|
Selection::Hexagon { cell, resolution } => {
|
||||||
|
let h3_res = h3o::Resolution::try_from(resolution)
|
||||||
|
.map_err(|err| format!("Invalid H3 resolution {resolution}: {err}"))?;
|
||||||
|
let need_parent = needs_parent(resolution);
|
||||||
|
let h3o_cell = h3o::CellIndex::try_from(cell)
|
||||||
|
.map_err(|err| format!("Invalid H3 cell: {err}"))?;
|
||||||
|
let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(h3o_cell, 0.001);
|
||||||
|
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
|
||||||
|
let mut seen: FxHashSet<&str> = FxHashSet::default();
|
||||||
|
let mut out: Vec<String> = Vec::new();
|
||||||
|
state.grid.for_each_in_bounds(
|
||||||
|
min_lat,
|
||||||
|
min_lon,
|
||||||
|
max_lat,
|
||||||
|
max_lon,
|
||||||
|
|row_idx| {
|
||||||
|
let row = row_idx as usize;
|
||||||
|
if cell_for_row_cached(
|
||||||
|
row,
|
||||||
|
&state.h3_cells,
|
||||||
|
h3_res,
|
||||||
|
need_parent,
|
||||||
|
&mut h3_cache,
|
||||||
|
) == cell
|
||||||
|
{
|
||||||
|
let pc = state.data.postcode(row);
|
||||||
|
if seen.insert(pc) {
|
||||||
|
out.push(pc.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let pc_refs: Vec<&str> = postcodes.iter().map(String::as_str).collect();
|
||||||
|
let indices = state.crime_records.gather(&pc_refs, since);
|
||||||
|
let total = indices.len();
|
||||||
|
let records: Vec<CrimeRecord> = indices
|
||||||
|
.iter()
|
||||||
|
.skip(offset)
|
||||||
|
.take(limit)
|
||||||
|
.map(|&idx| {
|
||||||
|
let v = state.crime_records.view(idx);
|
||||||
|
CrimeRecord {
|
||||||
|
month: format_month(v.month_index),
|
||||||
|
crime_type: v.crime_type.to_string(),
|
||||||
|
location: v.location.map(str::to_string),
|
||||||
|
outcome: v.outcome.map(str::to_string),
|
||||||
|
lat: v.lat,
|
||||||
|
lon: v.lon,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let truncated = offset + records.len() < total;
|
||||||
|
Ok(CrimeRecordsResponse {
|
||||||
|
records,
|
||||||
|
total,
|
||||||
|
offset,
|
||||||
|
truncated,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response())?
|
||||||
|
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error).into_response())?;
|
||||||
|
|
||||||
|
info!(total = result.total, returned = result.records.len(), "GET /api/crime-records");
|
||||||
|
Ok(Json(result))
|
||||||
|
}
|
||||||
|
|
@ -74,20 +74,20 @@ pub struct CrimeYearPoint {
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct CrimeYearStats {
|
pub struct CrimeYearStats {
|
||||||
/// Underlying crime type (e.g. "Burglary"). Matches existing crime feature
|
/// Underlying crime type, bare (e.g. "Burglary"). Matches the type prefix of
|
||||||
/// names with the `" (avg/yr)"` suffix stripped.
|
/// the `"(/yr, …)"` crime features.
|
||||||
pub name: String,
|
pub name: String,
|
||||||
pub points: Vec<CrimeYearPoint>,
|
pub points: Vec<CrimeYearPoint>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Average headline crime rate (avg/yr) for one crime type across the
|
/// Average crime count for one crime feature across the selection's outcode and
|
||||||
/// selection's outcode and postcode sector. Comparable to the national average
|
/// postcode sector. Comparable to the national average shown per metric in the
|
||||||
/// shown per metric in the right pane.
|
/// right pane.
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct CrimeAreaAverage {
|
pub struct CrimeAreaAverage {
|
||||||
/// Crime type, bare (e.g. "Burglary"). Matches `CrimeYearStats.name`.
|
/// Full crime-feature name (e.g. "Burglary (/yr, 7y)").
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// Exact national mean (avg/yr) — the frontend prefers this over the
|
/// Exact national mean count — the frontend prefers this over the
|
||||||
/// histogram-bin national average for crime so all four numbers in the row
|
/// histogram-bin national average for crime so all four numbers in the row
|
||||||
/// share one estimator.
|
/// share one estimator.
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
|
@ -161,10 +161,15 @@ pub struct HexagonStatsResponse {
|
||||||
/// present only when sector crime averages are available for it.
|
/// present only when sector crime averages are available for it.
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub crime_sector: Option<String>,
|
pub crime_sector: Option<String>,
|
||||||
/// Per-crime-type average rates across the central postcode's outcode and
|
/// Per-crime-type average counts across the central postcode's outcode and
|
||||||
/// sector, shown alongside the national average for each crime metric.
|
/// sector, shown alongside the national average for each crime metric.
|
||||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||||
pub crime_area_averages: Vec<CrimeAreaAverage>,
|
pub crime_area_averages: Vec<CrimeAreaAverage>,
|
||||||
|
/// Total individual crime records (last 7 years) across the distinct
|
||||||
|
/// postcodes in this selection — the count behind the "individual crimes"
|
||||||
|
/// list. Filter-independent, like `population`.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub crime_total_records: Option<u32>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub central_postcode: Option<String>,
|
pub central_postcode: Option<String>,
|
||||||
/// Total usual residents (ONS Census 2021) living across the distinct
|
/// Total usual residents (ONS Census 2021) living across the distinct
|
||||||
|
|
@ -699,26 +704,42 @@ pub async fn get_hexagon_stats(
|
||||||
&field_set,
|
&field_set,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Sum usual residents across the distinct postcodes covered by the
|
// Distinct postcodes covered by the hexagon, taken over `area_rows` (all
|
||||||
// hexagon. Computed over `area_rows` (all properties in the cell), not
|
// properties in the cell), not the filter-matching subset — population and
|
||||||
// the filter-matching subset, so toggling filters never changes it —
|
// the crime-records count are attributes of the area, independent of the
|
||||||
// population is an attribute of the area, like the council-house count.
|
// active filters (like the council-house count).
|
||||||
|
let mut seen: HashSet<&str> = HashSet::new();
|
||||||
|
let mut area_postcodes: Vec<&str> = Vec::new();
|
||||||
|
for &row in &area_rows {
|
||||||
|
let pc = state.data.postcode(row);
|
||||||
|
if seen.insert(pc) {
|
||||||
|
area_postcodes.push(pc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let population = {
|
let population = {
|
||||||
let mut seen: HashSet<&str> = HashSet::new();
|
|
||||||
let mut total: u64 = 0;
|
let mut total: u64 = 0;
|
||||||
let mut found = false;
|
let mut found = false;
|
||||||
for &row in &area_rows {
|
for &pc in &area_postcodes {
|
||||||
let pc = state.data.postcode(row);
|
if let Some(p) = state.population.for_postcode(pc) {
|
||||||
if seen.insert(pc) {
|
total += p as u64;
|
||||||
if let Some(p) = state.population.for_postcode(pc) {
|
found = true;
|
||||||
total += p as u64;
|
|
||||||
found = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
found.then(|| total.min(u32::MAX as u64) as u32)
|
found.then(|| total.min(u32::MAX as u64) as u32)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let crime_total_records = {
|
||||||
|
// Sum the per-postcode counts straight from the CSR index instead of
|
||||||
|
// materializing (and sorting) every record index: this keeps the
|
||||||
|
// mmap-backed columns cold on the hot hexagon path.
|
||||||
|
let total: u64 = area_postcodes
|
||||||
|
.iter()
|
||||||
|
.map(|pc| state.crime_records.total_for(pc) as u64)
|
||||||
|
.sum();
|
||||||
|
(total > 0).then(|| total.min(u32::MAX as u64) as u32)
|
||||||
|
};
|
||||||
|
|
||||||
Ok(HexagonStatsResponse {
|
Ok(HexagonStatsResponse {
|
||||||
count: total_count,
|
count: total_count,
|
||||||
numeric_features,
|
numeric_features,
|
||||||
|
|
@ -729,6 +750,7 @@ pub async fn get_hexagon_stats(
|
||||||
crime_outcode,
|
crime_outcode,
|
||||||
crime_sector,
|
crime_sector,
|
||||||
crime_area_averages,
|
crime_area_averages,
|
||||||
|
crime_total_records,
|
||||||
central_postcode,
|
central_postcode,
|
||||||
population,
|
population,
|
||||||
filter_exclusions,
|
filter_exclusions,
|
||||||
|
|
|
||||||
|
|
@ -227,6 +227,11 @@ pub async fn get_postcode_stats(
|
||||||
// Usual residents (Census 2021) for this postcode. Display-only.
|
// Usual residents (Census 2021) for this postcode. Display-only.
|
||||||
let population = state.population.for_postcode(&postcode_str);
|
let population = state.population.for_postcode(&postcode_str);
|
||||||
|
|
||||||
|
let crime_total_records = {
|
||||||
|
let total = state.crime_records.total_for(&postcode_str);
|
||||||
|
(total > 0).then_some(total)
|
||||||
|
};
|
||||||
|
|
||||||
Ok(HexagonStatsResponse {
|
Ok(HexagonStatsResponse {
|
||||||
count: total_count,
|
count: total_count,
|
||||||
numeric_features,
|
numeric_features,
|
||||||
|
|
@ -237,6 +242,7 @@ pub async fn get_postcode_stats(
|
||||||
crime_outcode,
|
crime_outcode,
|
||||||
crime_sector,
|
crime_sector,
|
||||||
crime_area_averages,
|
crime_area_averages,
|
||||||
|
crime_total_records,
|
||||||
central_postcode: None,
|
central_postcode: None,
|
||||||
population,
|
population,
|
||||||
filter_exclusions,
|
filter_exclusions,
|
||||||
|
|
|
||||||
|
|
@ -127,15 +127,20 @@ fn is_allowed_param_key(key: &str) -> bool {
|
||||||
| "filter"
|
| "filter"
|
||||||
| "school"
|
| "school"
|
||||||
| "crime"
|
| "crime"
|
||||||
|
| "crimeSeverity"
|
||||||
| "voteShare"
|
| "voteShare"
|
||||||
| "ethnicity"
|
| "ethnicity"
|
||||||
|
| "qualification"
|
||||||
|
| "tenure"
|
||||||
| "amenityDistance"
|
| "amenityDistance"
|
||||||
| "transportDistance"
|
| "transportDistance"
|
||||||
| "amenityCount2km"
|
| "amenityCount2km"
|
||||||
| "amenityCount5km"
|
| "amenityCount5km"
|
||||||
| "poi"
|
| "poi"
|
||||||
| "overlay"
|
| "overlay"
|
||||||
|
| "crimeType"
|
||||||
| "basemap"
|
| "basemap"
|
||||||
|
| "colorOpacity"
|
||||||
| "tab"
|
| "tab"
|
||||||
| "pc"
|
| "pc"
|
||||||
| "tt"
|
| "tt"
|
||||||
|
|
@ -594,6 +599,22 @@ mod tests {
|
||||||
assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
|
assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn preserves_all_filter_params_for_share_links() {
|
||||||
|
// Every filter param emitted by the frontend's stateToParams() must survive
|
||||||
|
// shortening; an unsupported key is rejected outright (see is_allowed_param_key),
|
||||||
|
// which fails the whole share link rather than dropping a single filter.
|
||||||
|
// Values use %3A (":") since form re-serialization keeps it stable.
|
||||||
|
let query = "crimeSeverity=Serious%3A0%3A5\
|
||||||
|
&qualification=Degree%3A20%3A80\
|
||||||
|
&tenure=Owner%3A30%3A90\
|
||||||
|
&crimeType=burglary\
|
||||||
|
&colorOpacity=60";
|
||||||
|
let params = sanitized_query_params(query, false).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(params, query);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn escapes_html_attributes() {
|
fn escapes_html_attributes() {
|
||||||
assert_eq!(escape_attr(r#""'><&"#), ""'><&");
|
assert_eq!(escape_attr(r#""'><&"#), ""'><&");
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,8 @@ use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
|
||||||
use crate::utils::{postcode_outcode, postcode_sector};
|
use crate::utils::{postcode_outcode, postcode_sector};
|
||||||
|
|
||||||
use super::hexagon_stats::{
|
use super::hexagon_stats::{
|
||||||
CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats,
|
CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats,
|
||||||
NumericFeatureStats, PricePoint,
|
HistogramStats, NumericFeatureStats, PricePoint,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
|
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
|
||||||
|
|
@ -352,11 +352,14 @@ pub fn compute_crime_by_year(
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() {
|
for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() {
|
||||||
// Crime types in the by-year side table are bare (e.g. "Burglary"), while
|
// Crime types in the by-year side table are bare (e.g. "Burglary"), while
|
||||||
// the configured feature names carry an " (avg/yr)" suffix. Match either
|
// the configured feature names carry a window suffix ("Burglary (/yr,
|
||||||
// form so callers can pass the feature names they already know.
|
// 7y)"). Emit the bare-type trend if the bare name is requested directly or
|
||||||
|
// any of its windowed features is.
|
||||||
if fields_specified {
|
if fields_specified {
|
||||||
let with_suffix = format!("{name} (avg/yr)");
|
let prefix = format!("{name} (");
|
||||||
if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
|
if !field_set.contains(name.as_str())
|
||||||
|
&& !field_set.iter().any(|f| f.starts_with(&prefix))
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -395,6 +398,7 @@ pub fn compute_crime_by_year(
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Latest year present anywhere in the by-year crime dataset. The client
|
/// Latest year present anywhere in the by-year crime dataset. The client
|
||||||
/// compares each selection's last charted year against this to caption
|
/// compares each selection's last charted year against this to caption
|
||||||
/// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as
|
/// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as
|
||||||
|
|
@ -440,13 +444,10 @@ pub fn area_crime_averages_for(
|
||||||
|
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for (idx, name) in averages.crime_types.iter().enumerate() {
|
for (idx, name) in averages.crime_types.iter().enumerate() {
|
||||||
// Crime types are bare here ("Burglary"); requested fields may carry the
|
// `name` is the full crime-feature name here (e.g. "Burglary (/yr,
|
||||||
// " (avg/yr)" suffix — accept either form, like compute_crime_by_year.
|
// 7y)"), matching exactly the feature fields the caller requests.
|
||||||
if fields_specified {
|
if fields_specified && !field_set.contains(name.as_str()) {
|
||||||
let with_suffix = format!("{name} (avg/yr)");
|
continue;
|
||||||
if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
let national_val = finite_at(Some(&averages.national), idx);
|
let national_val = finite_at(Some(&averages.national), idx);
|
||||||
let outcode_val = finite_at(outcode_means, idx);
|
let outcode_val = finite_at(outcode_means, idx);
|
||||||
|
|
@ -595,7 +596,10 @@ mod tests {
|
||||||
let mut by_sector = rustc_hash::FxHashMap::default();
|
let mut by_sector = rustc_hash::FxHashMap::default();
|
||||||
by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
|
by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
|
||||||
AreaCrimeAverages {
|
AreaCrimeAverages {
|
||||||
crime_types: vec!["Burglary".to_string(), "Robbery".to_string()],
|
crime_types: vec![
|
||||||
|
"Burglary (/yr, 7y)".to_string(),
|
||||||
|
"Robbery (/yr, 7y)".to_string(),
|
||||||
|
],
|
||||||
national: vec![8.0, 6.0],
|
national: vec![8.0, 6.0],
|
||||||
by_outcode,
|
by_outcode,
|
||||||
by_sector,
|
by_sector,
|
||||||
|
|
@ -611,12 +615,18 @@ mod tests {
|
||||||
assert_eq!(sector.as_deref(), Some("E14 2"));
|
assert_eq!(sector.as_deref(), Some("E14 2"));
|
||||||
assert_eq!(out.len(), 2);
|
assert_eq!(out.len(), 2);
|
||||||
|
|
||||||
let burglary = out.iter().find(|c| c.name == "Burglary").unwrap();
|
let burglary = out
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.name == "Burglary (/yr, 7y)")
|
||||||
|
.unwrap();
|
||||||
assert_eq!(burglary.national, Some(8.0));
|
assert_eq!(burglary.national, Some(8.0));
|
||||||
assert_eq!(burglary.outcode, Some(10.0));
|
assert_eq!(burglary.outcode, Some(10.0));
|
||||||
assert_eq!(burglary.sector, Some(5.0));
|
assert_eq!(burglary.sector, Some(5.0));
|
||||||
|
|
||||||
let robbery = out.iter().find(|c| c.name == "Robbery").unwrap();
|
let robbery = out
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.name == "Robbery (/yr, 7y)")
|
||||||
|
.unwrap();
|
||||||
assert_eq!(robbery.national, Some(6.0));
|
assert_eq!(robbery.national, Some(6.0));
|
||||||
// The outcode value was NaN — dropped to None; the sector value is finite.
|
// The outcode value was NaN — dropped to None; the sector value is finite.
|
||||||
assert_eq!(robbery.outcode, None);
|
assert_eq!(robbery.outcode, None);
|
||||||
|
|
@ -626,11 +636,13 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn area_crime_averages_respect_fields_filter() {
|
fn area_crime_averages_respect_fields_filter() {
|
||||||
let avgs = sample_averages();
|
let avgs = sample_averages();
|
||||||
// The suffixed feature-name form is accepted, like compute_crime_by_year.
|
// Area averages are keyed by the full crime-feature name.
|
||||||
let fields: HashSet<String> = ["Burglary (avg/yr)".to_string()].into_iter().collect();
|
let fields: HashSet<String> = ["Burglary (/yr, 7y)".to_string()]
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
|
let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
|
||||||
assert_eq!(out.len(), 1);
|
assert_eq!(out.len(), 1);
|
||||||
assert_eq!(out[0].name, "Burglary");
|
assert_eq!(out[0].name, "Burglary (/yr, 7y)");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -6,9 +6,9 @@ use rustc_hash::FxHashMap;
|
||||||
use crate::auth::TokenCache;
|
use crate::auth::TokenCache;
|
||||||
use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
|
use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
|
||||||
use crate::data::{
|
use crate::data::{
|
||||||
ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData,
|
ActualListingData, AreaCrimeAverages, CrimeByYearData, CrimeRecords,
|
||||||
POICategoryGroup, POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData,
|
DevelopmentData, OutcodeData, POICategoryGroup, POIData, PlaceData, PostcodeData,
|
||||||
TravelTimeStore,
|
PostcodePopulation, PropertyData, TravelTimeStore,
|
||||||
};
|
};
|
||||||
use crate::licensing::ShareBoundsCache;
|
use crate::licensing::ShareBoundsCache;
|
||||||
use crate::pocketbase::SuperuserTokenCache;
|
use crate::pocketbase::SuperuserTokenCache;
|
||||||
|
|
@ -52,10 +52,13 @@ pub struct AppState {
|
||||||
pub developments: Arc<DevelopmentData>,
|
pub developments: Arc<DevelopmentData>,
|
||||||
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
|
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
|
||||||
pub crime_by_year: Arc<CrimeByYearData>,
|
pub crime_by_year: Arc<CrimeByYearData>,
|
||||||
|
/// Per-postcode individual crime records (last 7 years), spill-backed,
|
||||||
|
/// served by the `/api/crime-records` endpoint and counted in stats.
|
||||||
|
pub crime_records: Arc<CrimeRecords>,
|
||||||
/// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
|
/// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
|
||||||
/// right pane. Display-only — never filterable. Empty when no data is loaded.
|
/// right pane. Display-only — never filterable. Empty when no data is loaded.
|
||||||
pub population: Arc<PostcodePopulation>,
|
pub population: Arc<PostcodePopulation>,
|
||||||
/// Precomputed per-outcode and per-postcode-sector average crime rates,
|
/// Precomputed per-outcode and per-postcode-sector average crime counts,
|
||||||
/// shown in the right pane alongside the national average for each metric.
|
/// shown in the right pane alongside the national average for each metric.
|
||||||
pub area_crime_averages: Arc<AreaCrimeAverages>,
|
pub area_crime_averages: Arc<AreaCrimeAverages>,
|
||||||
/// Token validation cache (60s TTL)
|
/// Token validation cache (60s TTL)
|
||||||
|
|
@ -178,6 +181,7 @@ impl AppState {
|
||||||
series_by_postcode: FxHashMap::default(),
|
series_by_postcode: FxHashMap::default(),
|
||||||
covered_years_by_postcode: FxHashMap::default(),
|
covered_years_by_postcode: FxHashMap::default(),
|
||||||
}),
|
}),
|
||||||
|
crime_records: Arc::new(CrimeRecords::empty()),
|
||||||
population: Arc::new(PostcodePopulation::empty()),
|
population: Arc::new(PostcodePopulation::empty()),
|
||||||
area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
|
area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
|
||||||
token_cache: Arc::new(TokenCache::new()),
|
token_cache: Arc::new(TokenCache::new()),
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue