+
)}
diff --git a/frontend/src/components/map/filters/SpecificCrimeFilterCard.tsx b/frontend/src/components/map/filters/VariantFilterCard.tsx
similarity index 51%
rename from frontend/src/components/map/filters/SpecificCrimeFilterCard.tsx
rename to frontend/src/components/map/filters/VariantFilterCard.tsx
index 07273ba..325e9fd 100644
--- a/frontend/src/components/map/filters/SpecificCrimeFilterCard.tsx
+++ b/frontend/src/components/map/filters/VariantFilterCard.tsx
@@ -5,23 +5,22 @@ import { ChevronIcon } from '../../ui/icons';
import { FeatureActions } from '../../ui/FeatureIcons';
import { FeatureLabel } from '../../ui/FeatureLabel';
import type { FeatureFilters, FeatureMeta } from '../../../types';
+import type { VariantFilterConfig } from '../../../lib/variant-filter';
import { formatNumber, type PercentileScale } from '../../../lib/format';
import { getFeatureIcon } from '../../../lib/feature-icons';
import { getGroupIcon } from '../../../lib/group-icons';
-import {
- SPECIFIC_CRIMES_FILTER_NAME,
- SPECIFIC_CRIME_FEATURE_NAMES,
- clampSpecificCrimeRange,
- getDefaultSpecificCrimeFeatureName,
- getSpecificCrimeFeatureName,
- getSpecificCrimeFilterMeta,
- replaceSpecificCrimeFilterKeySelection,
-} from '../../../lib/crime-filter';
import { SliderLabels } from './SliderLabels';
-export function SpecificCrimeFilterCard({
+/**
+ * One card for a "pick a variant, filter its range" aggregate filter
+ * (specific crimes, qualifications, …). Behaviour is variant-agnostic: the
+ * `config` supplies the dropdown options, labels and helper functions so the
+ * same component backs every such filter (see [`VariantFilterConfig`]).
+ */
+export function VariantFilterCard({
+ config,
features,
- crimeFeature,
+ variantFeature,
filters,
activeFeature,
dragValue,
@@ -36,8 +35,9 @@ export function SpecificCrimeFilterCard({
onShowInfo,
onRemove,
}: {
+ config: VariantFilterConfig;
features: FeatureMeta[];
- crimeFeature: FeatureMeta;
+ variantFeature: FeatureMeta;
filters: FeatureFilters;
activeFeature: string | null;
dragValue: [number, number] | null;
@@ -53,27 +53,62 @@ export function SpecificCrimeFilterCard({
onRemove: () => void;
}) {
const { t } = useTranslation();
- const specificCrimeMeta = getSpecificCrimeFilterMeta(features);
- const crimeOptions = SPECIFIC_CRIME_FEATURE_NAMES.map((name) =>
- features.find((feature) => feature.name === name)
- ).filter((feature): feature is FeatureMeta => Boolean(feature));
+ const variantMeta = config.getFilterMeta(features);
const selectedFeatureName =
- getSpecificCrimeFeatureName(crimeFeature.name) ?? getDefaultSpecificCrimeFeatureName(features);
+ config.getFeatureName(variantFeature.name) ?? config.getDefaultFeatureName(features);
const selectedFeature = selectedFeatureName
? features.find((feature) => feature.name === selectedFeatureName)
: undefined;
- if (!selectedFeature || crimeOptions.length === 0 || !selectedFeatureName) return null;
+ // Optional secondary axis: the same variant measured over a different window
+ // (e.g. 7- vs 2-year crime rates). The current window comes from the selected
+ // feature so the dropdown can re-point each option into that window.
+ const windowConfig = config.window;
+ const currentWindow =
+ (windowConfig && selectedFeatureName ? windowConfig.getWindow(selectedFeatureName) : null) ??
+ windowConfig?.options[0]?.id ??
+ null;
- const isActive = activeFeature === crimeFeature.name;
- const isPinned = pinnedFeature === crimeFeature.name;
+ const variantOptions = config.featureNames
+ .map((canonicalName) => {
+ const optionName =
+ windowConfig && currentWindow
+ ? windowConfig.withWindow(canonicalName, currentWindow)
+ : canonicalName;
+ const feature = features.find((f) => f.name === optionName);
+ if (!feature) return null;
+ return {
+ value: optionName,
+ label: ts(config.getOptionLabelSource ? config.getOptionLabelSource(optionName) : optionName),
+ feature,
+ };
+ })
+ .filter((option): option is { value: string; label: string; feature: FeatureMeta } =>
+ Boolean(option)
+ );
+
+ // Only offer windows that actually exist for the selected variant, so a
+ // missing backend column can't strand the card on an empty selection.
+ const windowOptions =
+ windowConfig && selectedFeatureName
+ ? windowConfig.options.filter((option) =>
+ features.some(
+ (f) => f.name === windowConfig.withWindow(selectedFeatureName, option.id)
+ )
+ )
+ : [];
+
+ if (!selectedFeature || variantOptions.length === 0 || !selectedFeatureName) return null;
+
+ const isActive = activeFeature === variantFeature.name;
+ const isPinned = pinnedFeature === variantFeature.name;
const hist = selectedFeature.histogram;
const dataMin = hist?.min ?? selectedFeature.min ?? 0;
const dataMax = hist?.max ?? selectedFeature.max ?? 100;
const displayValue =
isActive && dragValue
? dragValue
- : (filters[crimeFeature.name] as [number, number]) || [dataMin, dataMax];
+ : (filters[variantFeature.name] as [number, number]) || [dataMin, dataMax];
const scale = percentileScale;
const clampMin = displayValue[0] <= dataMin;
const clampMax = displayValue[1] >= dataMax;
@@ -89,14 +124,14 @@ export function SpecificCrimeFilterCard({
clampMax ? (selectedFeature.max ?? dataMax) : displayValue[1],
];
- const replaceCrimeFeature = (nextFeatureName: string) => {
- const nextName = replaceSpecificCrimeFilterKeySelection(crimeFeature.name, nextFeatureName);
- if (nextName === crimeFeature.name) return;
+ const replaceVariantFeature = (nextFeatureName: string) => {
+ const nextName = config.replaceFilterKeySelection(variantFeature.name, nextFeatureName);
+ if (nextName === variantFeature.name) return;
const nextFeature = features.find((feature) => feature.name === nextFeatureName);
const nextDataMin = nextFeature?.histogram?.min ?? nextFeature?.min ?? 0;
const nextDataMax = nextFeature?.histogram?.max ?? nextFeature?.max ?? Math.max(1, dataMax);
- const nextRange = clampSpecificCrimeRange(
+ const nextRange = config.clampRange(
[
displayValue[0] <= dataMin ? nextDataMin : displayValue[0],
displayValue[1] >= dataMax ? nextDataMax : displayValue[1],
@@ -108,6 +143,11 @@ export function SpecificCrimeFilterCard({
if (isPinned) onTogglePin(nextName);
};
+ const switchWindow = (windowId: string) => {
+ if (!windowConfig || windowId === currentWindow) return;
+ replaceVariantFeature(windowConfig.withWindow(selectedFeatureName, windowId));
+ };
+
const mobileIconClass = 'w-4 h-4 text-teal-600 dark:text-teal-400 shrink-0';
const mobileIcon =
getFeatureIcon(selectedFeature.name, mobileIconClass) ||
@@ -118,7 +158,7 @@ export function SpecificCrimeFilterCard({
return (
-
-
- {t('filters.crimeType')}
-
-
-
replaceCrimeFeature(e.target.value)}
- className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50"
- >
- {crimeOptions.map((option) => (
-
- {ts(option.name)}
-
- ))}
-
-
+ {/* A single-variant filter (e.g. Serious/Minor crime) has nothing to pick,
+ so the dropdown is hidden and only the window toggle + slider remain. */}
+ {variantOptions.length > 1 && (
+
+
+ {t(config.dropdownLabelKey)}
+
+
+ replaceVariantFeature(e.target.value)}
+ className="w-full appearance-none rounded-md border border-warm-200 bg-warm-50 px-2 py-1.5 pr-8 text-sm font-medium text-navy-950 shadow-inner outline-none transition-colors hover:bg-white focus:border-teal-400 focus:ring-2 focus:ring-teal-200 dark:border-warm-700 dark:bg-navy-900 dark:text-warm-100 dark:hover:bg-navy-800 dark:focus:ring-teal-900/50"
+ >
+ {variantOptions.map((option) => (
+
+ {option.label}
+
+ ))}
+
+
-
+ )}
+
+ {windowConfig && currentWindow && windowOptions.length > 1 && (
+
+ {windowConfig.labelKey && (
+
+ {t(windowConfig.labelKey)}
+
+ )}
+
+ {windowOptions.map((option) => {
+ const active = option.id === currentWindow;
+ return (
+ switchWindow(option.id)}
+ className={`rounded px-2.5 py-1 text-xs font-medium transition-colors ${
+ active
+ ? 'bg-teal-600 text-white shadow-sm'
+ : 'text-warm-500 hover:text-navy-950 dark:text-warm-400 dark:hover:text-warm-100'
+ }`}
+ >
+ {t(option.labelKey)}
+
+ );
+ })}
+
+
+ )}
{mobileIcon &&
{mobileIcon}
}
@@ -198,7 +275,7 @@ export function SpecificCrimeFilterCard({
max >= (selectedFeature.max ?? dataMax) ? dataMax : max,
])
}
- onPointerDown={() => onDragStart(crimeFeature.name, displayValue)}
+ onPointerDown={() => onDragStart(variantFeature.name, displayValue)}
onPointerUp={() => onDragEnd()}
/>
- onFilterChange(crimeFeature.name, clampSpecificCrimeRange(v, selectedFeature))
+ onFilterChange(variantFeature.name, config.clampRange(v, selectedFeature))
}
/>
{filterImpact != null && filterImpact > 0 && (
diff --git a/frontend/src/components/map/map-page/DesktopMapPage.tsx b/frontend/src/components/map/map-page/DesktopMapPage.tsx
index f66cd5d..9f65db4 100644
--- a/frontend/src/components/map/map-page/DesktopMapPage.tsx
+++ b/frontend/src/components/map/map-page/DesktopMapPage.tsx
@@ -236,8 +236,8 @@ export function DesktopMapPage({
onClick={onToggleActualListings}
aria-pressed={actualListingsEnabled}
aria-busy={actualListingsLoading}
- aria-label={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
- title={actualListingsEnabled ? 'Hide actual listings' : 'Show actual listings'}
+ aria-label={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
+ title={actualListingsEnabled ? t('map.actualListings.hide') : t('map.actualListings.show')}
className={`flex items-center gap-2 rounded-lg bg-white px-3 py-2 shadow-lg dark:bg-warm-800 ${actualListingsEnabled ? 'text-red-600 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300' : 'text-warm-500 hover:text-red-600 dark:text-warm-400 dark:hover:text-red-400'}`}
>
{actualListingsLoading ? (
@@ -246,16 +246,17 @@ export function DesktopMapPage({
- Listings{actualListingsEnabled ? ` (${actualListings.length})` : ''}
+ {t('map.actualListings.label')}{actualListingsEnabled ? ` (${actualListings.length})` : ''}
)}
Overlays
+ {t('overlays.heading')}
{actualListingsLoading ? (
diff --git a/frontend/src/components/map/map-page/derivedState.ts b/frontend/src/components/map/map-page/derivedState.ts
index 388a466..040fb98 100644
--- a/frontend/src/components/map/map-page/derivedState.ts
+++ b/frontend/src/components/map/map-page/derivedState.ts
@@ -6,8 +6,11 @@ import type { HexagonLocation } from '../../../lib/external-search';
import type { useMapData } from '../../../hooks/useMapData';
import { resolveTransitVariant, type TravelTimeEntry } from '../../../hooks/useTravelTime';
import { getSpecificCrimeFeatureName } from '../../../lib/crime-filter';
+import { getCrimeSeverityFeatureName } from '../../../lib/crime-severity-filter';
import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
+import { getQualificationFeatureName } from '../../../lib/qualification-filter';
+import { getTenureFeatureName } from '../../../lib/tenure-filter';
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
@@ -23,8 +26,11 @@ export function getMapPageBackendFeatureName(featureName: string): string {
return (
getSchoolBackendFeatureName(featureName) ??
getSpecificCrimeFeatureName(featureName) ??
+ getCrimeSeverityFeatureName(featureName) ??
getElectionVoteShareFeatureName(featureName) ??
getEthnicityFeatureName(featureName) ??
+ getQualificationFeatureName(featureName) ??
+ getTenureFeatureName(featureName) ??
getPoiDistanceFeatureName(featureName) ??
featureName
);
diff --git a/frontend/src/components/ui/IndeterminateProgressBar.tsx b/frontend/src/components/ui/IndeterminateProgressBar.tsx
index 97e89e8..8774300 100644
--- a/frontend/src/components/ui/IndeterminateProgressBar.tsx
+++ b/frontend/src/components/ui/IndeterminateProgressBar.tsx
@@ -1,16 +1,20 @@
+import { useTranslation } from 'react-i18next';
+
interface IndeterminateProgressBarProps {
show: boolean;
className?: string;
}
export function IndeterminateProgressBar({ show, className = '' }: IndeterminateProgressBarProps) {
+ const { t } = useTranslation();
+
if (!show) return null;
return (
diff --git a/frontend/src/hooks/useAiFilters.ts b/frontend/src/hooks/useAiFilters.ts
index ae48c56..3bfd58c 100644
--- a/frontend/src/hooks/useAiFilters.ts
+++ b/frontend/src/hooks/useAiFilters.ts
@@ -1,4 +1,5 @@
import { useState, useCallback, useRef } from 'react';
+import { useTranslation } from 'react-i18next';
import type { FeatureFilters } from '../types';
import { apiUrl, authHeaders, logNonAbortError } from '../lib/api';
@@ -88,6 +89,7 @@ function buildSummary(
}
export function useAiFilters(): UseAiFiltersResult {
+ const { t } = useTranslation();
const [loading, setLoading] = useState(false);
const [error, setError] = useState
(null);
const [errorType, setErrorType] = useState(null);
@@ -170,14 +172,14 @@ export function useAiFilters(): UseAiFiltersResult {
} catch (err) {
if (controller.signal.aborted) return null;
logNonAbortError('ai-filters', err);
- const message = err instanceof Error ? err.message : 'Failed to generate filters';
+ const message = err instanceof Error ? err.message : t('aiFilter.generateFailed');
setErrorType('error');
setError(message);
setLoading(false);
return null;
}
},
- []
+ [t]
);
return { fetchAiFilters, loading, error, errorType, notes, summary };
diff --git a/frontend/src/hooks/useAuth.ts b/frontend/src/hooks/useAuth.ts
index 134cd2c..73c628c 100644
--- a/frontend/src/hooks/useAuth.ts
+++ b/frontend/src/hooks/useAuth.ts
@@ -1,4 +1,5 @@
import { useState, useEffect, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
import pb from '../lib/pocketbase';
import { trackEvent } from '../lib/analytics';
@@ -31,6 +32,7 @@ function authRefreshInvalidated(error: unknown): boolean {
}
export function useAuth() {
+ const { t } = useTranslation();
const [user, setUser] = useState(() => {
if (pb.authStore.isValid && pb.authStore.record) {
return recordToUser(pb.authStore.record);
@@ -60,13 +62,13 @@ export function useAuth() {
setUser(recordToUser(result.record));
trackEvent('Login', { method: 'email' });
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Login failed';
+ const msg = err instanceof Error ? err.message : t('auth.loginFailed');
setError(msg);
throw err;
} finally {
setLoading(false);
}
- }, []);
+ }, [t]);
const register = useCallback(async (email: string, password: string) => {
setLoading(true);
@@ -83,13 +85,13 @@ export function useAuth() {
setUser(recordToUser(result.record));
trackEvent('Register');
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Registration failed';
+ const msg = err instanceof Error ? err.message : t('auth.registrationFailed');
setError(msg);
throw err;
} finally {
setLoading(false);
}
- }, []);
+ }, [t]);
const loginWithOAuth = useCallback(async (provider: string) => {
setLoading(true);
@@ -102,13 +104,13 @@ export function useAuth() {
setUser(recordToUser(result.record));
trackEvent('Login', { method: provider });
} catch (err) {
- const msg = err instanceof Error ? err.message : 'OAuth login failed';
+ const msg = err instanceof Error ? err.message : t('auth.oauthFailed');
setError(msg);
throw err;
} finally {
setLoading(false);
}
- }, []);
+ }, [t]);
const logout = useCallback(() => {
trackEvent('Logout');
@@ -122,13 +124,13 @@ export function useAuth() {
try {
await pb.collection('users').requestPasswordReset(email);
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Password reset request failed';
+ const msg = err instanceof Error ? err.message : t('auth.passwordResetFailed');
setError(msg);
throw err;
} finally {
setLoading(false);
}
- }, []);
+ }, [t]);
const refreshAuth = useCallback(async () => {
setLoading(true);
diff --git a/frontend/src/hooks/useClickedListings.test.ts b/frontend/src/hooks/useClickedListings.test.ts
new file mode 100644
index 0000000..78ff75e
--- /dev/null
+++ b/frontend/src/hooks/useClickedListings.test.ts
@@ -0,0 +1,98 @@
+import { act, renderHook, waitFor } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mocks = vi.hoisted(() => ({
+ create: vi.fn(),
+ getFullList: vi.fn(),
+ authStore: {
+ isValid: true,
+ record: { id: 'user-1' } as { id: string } | null,
+ onChange: () => () => {},
+ },
+}));
+
+vi.mock('../lib/pocketbase', () => ({
+ default: {
+ authStore: mocks.authStore,
+ collection: () => ({ getFullList: mocks.getFullList, create: mocks.create }),
+ },
+}));
+
+import { useClickedListings } from './useClickedListings';
+
+describe('useClickedListings', () => {
+ beforeEach(() => {
+ mocks.create.mockReset().mockResolvedValue({});
+ mocks.getFullList.mockReset().mockResolvedValue([]);
+ mocks.authStore.isValid = true;
+ mocks.authStore.record = { id: 'user-1' };
+ });
+
+ afterEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it('hydrates the visited set from PocketBase for the signed-in user', async () => {
+ mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
+ const { result } = renderHook(() => useClickedListings());
+
+ await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
+ });
+
+ it('records a click optimistically and persists it', async () => {
+ const { result } = renderHook(() => useClickedListings());
+ await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
+
+ act(() => result.current.markClicked('https://example.com/b'));
+
+ // Visible immediately — no need to await the network round-trip.
+ expect(result.current.clickedUrls.has('https://example.com/b')).toBe(true);
+ expect(mocks.create).toHaveBeenCalledWith({
+ user: 'user-1',
+ url: 'https://example.com/b',
+ });
+ });
+
+ it('produces a new Set instance for a new url so deck.gl re-colours', async () => {
+ const { result } = renderHook(() => useClickedListings());
+ await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
+
+ const before = result.current.clickedUrls;
+ act(() => result.current.markClicked('https://example.com/c'));
+ expect(result.current.clickedUrls).not.toBe(before);
+ });
+
+ it('does not re-persist an already visited url', async () => {
+ mocks.getFullList.mockResolvedValue([{ url: 'https://example.com/a' }]);
+ const { result } = renderHook(() => useClickedListings());
+ await waitFor(() => expect(result.current.clickedUrls.has('https://example.com/a')).toBe(true));
+
+ const before = result.current.clickedUrls;
+ act(() => result.current.markClicked('https://example.com/a'));
+
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(result.current.clickedUrls).toBe(before); // identity stable → no needless recompute
+ });
+
+ it('ignores empty urls', async () => {
+ const { result } = renderHook(() => useClickedListings());
+ await waitFor(() => expect(mocks.getFullList).toHaveBeenCalled());
+
+ act(() => result.current.markClicked(''));
+ act(() => result.current.markClicked(undefined));
+
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(result.current.clickedUrls.size).toBe(0);
+ });
+
+ it('recolours for anonymous users in-memory but does not persist', async () => {
+ mocks.authStore.isValid = false;
+ mocks.authStore.record = null;
+ const { result } = renderHook(() => useClickedListings());
+
+ act(() => result.current.markClicked('https://example.com/d'));
+
+ expect(result.current.clickedUrls.has('https://example.com/d')).toBe(true);
+ expect(mocks.create).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/src/hooks/useClickedListings.ts b/frontend/src/hooks/useClickedListings.ts
new file mode 100644
index 0000000..ae28fd0
--- /dev/null
+++ b/frontend/src/hooks/useClickedListings.ts
@@ -0,0 +1,82 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+import pb from '../lib/pocketbase';
+
+/** Per-user record of opened listings, stored in PocketBase so visited listings can be drawn
+ * in a distinct colour on the map and persist across devices. One row per (user, url). */
+const CLICKED_LISTINGS_COLLECTION = 'clicked_listings';
+
+function currentUserId(): string | null {
+ return pb.authStore.isValid && pb.authStore.record ? pb.authStore.record.id : null;
+}
+
+/** Tracks which listings the signed-in user has opened.
+ *
+ * Reads the user from the shared PocketBase authStore so it stays self-contained — no need to
+ * thread a user id down through the map layer tree. Returns a Set for O(1) membership checks in
+ * the map's colour accessor; a fresh Set instance is produced on every change so it can double
+ * as a deck.gl `updateTrigger` (identity-compared).
+ *
+ * Writes are optimistic: `markClicked` updates local state synchronously so the pin recolours
+ * the instant it is clicked, independent of the PocketBase round-trip or the next listings fetch.
+ */
+export function useClickedListings() {
+ const [clickedUrls, setClickedUrls] = useState>(() => new Set());
+ const [userId, setUserId] = useState(currentUserId);
+
+ const userIdRef = useRef(userId);
+ userIdRef.current = userId;
+ // Mirrors the latest committed set so markClicked can decide (synchronously) whether a url is
+ // new — without depending on the async state updater having run yet.
+ const clickedUrlsRef = useRef(clickedUrls);
+ clickedUrlsRef.current = clickedUrls;
+
+ // Follow login/logout via the shared authStore (also covers cross-tab auth changes).
+ useEffect(() => {
+ const unsubscribe = pb.authStore.onChange(() => setUserId(currentUserId()));
+ return unsubscribe;
+ }, []);
+
+ // Load the user's previously opened listings whenever the signed-in user changes. Signed-out
+ // users keep an in-memory set for the current session only (nothing to persist to).
+ useEffect(() => {
+ if (!userId) {
+ setClickedUrls(new Set());
+ return;
+ }
+ let cancelled = false;
+ pb.collection(CLICKED_LISTINGS_COLLECTION)
+ .getFullList({ filter: `user = "${userId}"`, fields: 'url' })
+ .then((records) => {
+ if (cancelled) return;
+ const urls = records
+ .map((record) => (record as { url?: unknown }).url)
+ .filter((url): url is string => typeof url === 'string');
+ setClickedUrls(new Set(urls));
+ })
+ .catch(() => {
+ // Visited history is a convenience; a failed load just starts the session empty.
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [userId]);
+
+ const markClicked = useCallback((url: string | undefined | null) => {
+ if (!url || clickedUrlsRef.current.has(url)) return; // unknown or already visited
+
+ // Optimistic, synchronous local update → the map pin recolours on click.
+ setClickedUrls((prev) => (prev.has(url) ? prev : new Set(prev).add(url)));
+
+ const uid = userIdRef.current;
+ if (!uid) return; // anonymous: recolour for the session, nothing to persist
+ pb.collection(CLICKED_LISTINGS_COLLECTION)
+ .create({ user: uid, url })
+ .catch(() => {
+ // Ignore duplicate-index conflicts and transient failures — local state already
+ // reflects the click, and a missed write only means it won't persist to the next visit.
+ });
+ }, []);
+
+ return { clickedUrls, markClicked };
+}
diff --git a/frontend/src/hooks/useDeckLayers.ts b/frontend/src/hooks/useDeckLayers.ts
index b647de9..25a3640 100644
--- a/frontend/src/hooks/useDeckLayers.ts
+++ b/frontend/src/hooks/useDeckLayers.ts
@@ -17,6 +17,8 @@ import type {
import {
DENSITY_GRADIENT,
DENSITY_GRADIENT_DARK,
+ FILTERED_OUT_FILL,
+ FILTERED_OUT_LINE,
getEnumPaletteForFeature,
getFeatureGradient,
} from '../lib/consts';
@@ -125,11 +127,12 @@ export function useDeckLayers({
zoom,
isDark,
});
- const { listingLayers, listingPopup, clearListingPopup } = useListingLayers({
- listings: actualListings,
- zoom,
- isDark,
- });
+ const { listingLayers, listingPopup, clearListingPopup, clickedListingUrls, markListingClicked } =
+ useListingLayers({
+ listings: actualListings,
+ zoom,
+ isDark,
+ });
const { developmentLayers, developmentPopup, clearDevelopmentPopup } = useDevelopmentLayers({
developments,
zoom,
@@ -350,7 +353,12 @@ export function useDeckLayers({
getHexagon: (d) => d.h3,
getFillColor: (d) => {
if ((d.count as number) <= 0) {
- return [0, 0, 0, 0] as [number, number, number, number];
+ // Filtered out: keep it on the map as a faint grey ghost (still
+ // clickable) rather than hiding it entirely.
+ return withColorOpacity(
+ FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
+ colorOpacityRef.current
+ );
}
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
const dark = isDarkRef.current;
@@ -504,7 +512,12 @@ export function useDeckLayers({
getFillColor: (f) => {
const d = f.properties;
if (d.count <= 0) {
- return [0, 0, 0, 0] as [number, number, number, number];
+ // Filtered out: keep it on the map as a faint grey ghost (still
+ // clickable) rather than hiding it entirely.
+ return withColorOpacity(
+ FILTERED_OUT_FILL[isDarkRef.current ? 'dark' : 'light'],
+ colorOpacityRef.current
+ );
}
const fill = (color: RgbaColor) => withColorOpacity(color, colorOpacityRef.current);
const dark = isDarkRef.current;
@@ -587,7 +600,7 @@ export function useDeckLayers({
return [29, 228, 195, 255] as [number, number, number, number];
if (pc === hoveredPostcodeRef.current)
return [29, 228, 195, 200] as [number, number, number, number];
- if (f.properties.count <= 0) return [0, 0, 0, 0] as [number, number, number, number];
+ if (f.properties.count <= 0) return FILTERED_OUT_LINE[dark ? 'dark' : 'light'];
return (dark ? [180, 170, 160, 100] : [100, 100, 100, 150]) as [
number,
number,
@@ -599,7 +612,7 @@ export function useDeckLayers({
const pc = f.properties.postcode;
if (pc === selectedPostcodeIdRef.current) return 4;
if (pc === hoveredPostcodeRef.current) return 2;
- if (f.properties.count <= 0) return 0;
+ // Filtered-out polygons (count <= 0) keep the same 1px ghost border.
return 1;
},
lineWidthUnits: 'pixels',
@@ -749,6 +762,8 @@ export function useDeckLayers({
clearPopupInfo,
listingPopup,
clearListingPopup,
+ clickedListingUrls,
+ markListingClicked,
developmentPopup,
clearDevelopmentPopup,
hoverPosition,
diff --git a/frontend/src/hooks/useFilters.ts b/frontend/src/hooks/useFilters.ts
index 44ca798..6249419 100644
--- a/frontend/src/hooks/useFilters.ts
+++ b/frontend/src/hooks/useFilters.ts
@@ -17,6 +17,15 @@ import {
getSpecificCrimeFilterKeyId,
normalizeSpecificCrimeFilters,
} from '../lib/crime-filter';
+import {
+ CRIME_SEVERITY_FILTER_NAMES,
+ createCrimeSeverityFilterKey,
+ getCrimeSeverityFeatureName,
+ getCrimeSeverityFilterKeyId,
+ getDefaultCrimeSeverityFeatureName,
+ normalizeCrimeSeverityFilters,
+ type CrimeSeverityFilterName,
+} from '../lib/crime-severity-filter';
import {
ELECTION_VOTE_SHARE_FILTER_NAME,
createElectionVoteShareFilterKey,
@@ -33,6 +42,22 @@ import {
getEthnicityFilterKeyId,
normalizeEthnicityFilters,
} from '../lib/ethnicity-filter';
+import {
+ QUALIFICATIONS_FILTER_NAME,
+ createQualificationFilterKey,
+ getDefaultQualificationFeatureName,
+ getQualificationFeatureName,
+ getQualificationFilterKeyId,
+ normalizeQualificationFilters,
+} from '../lib/qualification-filter';
+import {
+ TENURE_FILTER_NAME,
+ createTenureFilterKey,
+ getDefaultTenureFeatureName,
+ getTenureFeatureName,
+ getTenureFilterKeyId,
+ normalizeTenureFilters,
+} from '../lib/tenure-filter';
import {
POI_FILTER_NAMES,
createPoiFilterKey,
@@ -55,9 +80,15 @@ interface UseFiltersOptions {
function normalizeFilters(filters: FeatureFilters): FeatureFilters {
return normalizePoiDistanceFilters(
- normalizeEthnicityFilters(
- normalizeElectionVoteShareFilters(
- normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters))
+ normalizeTenureFilters(
+ normalizeQualificationFilters(
+ normalizeEthnicityFilters(
+ normalizeElectionVoteShareFilters(
+ normalizeCrimeSeverityFilters(
+ normalizeSpecificCrimeFilters(normalizeSchoolFilters(filters))
+ )
+ )
+ )
)
)
);
@@ -67,8 +98,11 @@ function getBackendFeatureName(name: string): string {
return (
getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ??
+ getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ??
+ getQualificationFeatureName(name) ??
+ getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ??
name
);
@@ -140,12 +174,21 @@ export function useFilters({
const specificCrimeFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getSpecificCrimeFilterKeyId)
);
+ const crimeSeverityFilterIdRef = useRef(
+ getNextNumericKeyId(initialFiltersRef.current!, getCrimeSeverityFilterKeyId)
+ );
const electionVoteShareFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getElectionVoteShareFilterKeyId)
);
const ethnicityFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getEthnicityFilterKeyId)
);
+ const qualificationFilterIdRef = useRef(
+ getNextNumericKeyId(initialFiltersRef.current!, getQualificationFilterKeyId)
+ );
+ const tenureFilterIdRef = useRef(
+ getNextNumericKeyId(initialFiltersRef.current!, getTenureFilterKeyId)
+ );
const poiDistanceFilterIdRef = useRef(
getNextNumericKeyId(initialFiltersRef.current!, getPoiDistanceFilterKeyId)
);
@@ -184,8 +227,11 @@ export function useFilters({
if (
name !== SCHOOL_FILTER_NAME &&
name !== SPECIFIC_CRIMES_FILTER_NAME &&
+ !CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) &&
name !== ELECTION_VOTE_SHARE_FILTER_NAME &&
name !== ETHNICITIES_FILTER_NAME &&
+ name !== QUALIFICATIONS_FILTER_NAME &&
+ name !== TENURE_FILTER_NAME &&
!POI_FILTER_NAMES.includes(name as PoiFilterName) &&
!meta
) {
@@ -201,8 +247,11 @@ export function useFilters({
const addsNewKey =
name === SCHOOL_FILTER_NAME ||
name === SPECIFIC_CRIMES_FILTER_NAME ||
+ CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName) ||
name === ELECTION_VOTE_SHARE_FILTER_NAME ||
name === ETHNICITIES_FILTER_NAME ||
+ name === QUALIFICATIONS_FILTER_NAME ||
+ name === TENURE_FILTER_NAME ||
POI_FILTER_NAMES.includes(name as PoiFilterName) ||
!(name in current);
if (addsNewKey && Object.keys(current).length >= limit) {
@@ -247,6 +296,29 @@ export function useFilters({
],
};
}
+ if (CRIME_SEVERITY_FILTER_NAMES.includes(name as CrimeSeverityFilterName)) {
+ const severityFilterName = name as CrimeSeverityFilterName;
+ const defaultSeverityFeatureName = getDefaultCrimeSeverityFeatureName(
+ features,
+ severityFilterName
+ );
+ const defaultSeverityFeature = defaultSeverityFeatureName
+ ? features.find((feature) => feature.name === defaultSeverityFeatureName)
+ : undefined;
+ if (!defaultSeverityFeatureName) return prev;
+
+ return {
+ ...prev,
+ [createCrimeSeverityFilterKey(
+ severityFilterName,
+ defaultSeverityFeatureName,
+ crimeSeverityFilterIdRef.current++
+ )]: [
+ defaultSeverityFeature?.histogram?.min ?? defaultSeverityFeature?.min ?? 0,
+ defaultSeverityFeature?.histogram?.max ?? defaultSeverityFeature?.max ?? 100,
+ ],
+ };
+ }
if (name === ELECTION_VOTE_SHARE_FILTER_NAME) {
const defaultVoteShareFeatureName = getDefaultElectionVoteShareFeatureName(features);
const defaultVoteShareFeature = defaultVoteShareFeatureName
@@ -281,6 +353,41 @@ export function useFilters({
],
};
}
+ if (name === QUALIFICATIONS_FILTER_NAME) {
+ const defaultQualificationFeatureName = getDefaultQualificationFeatureName(features);
+ const defaultQualificationFeature = defaultQualificationFeatureName
+ ? features.find((feature) => feature.name === defaultQualificationFeatureName)
+ : undefined;
+ if (!defaultQualificationFeatureName) return prev;
+
+ return {
+ ...prev,
+ [createQualificationFilterKey(
+ defaultQualificationFeatureName,
+ qualificationFilterIdRef.current++
+ )]: [
+ defaultQualificationFeature?.histogram?.min ?? defaultQualificationFeature?.min ?? 0,
+ defaultQualificationFeature?.histogram?.max ??
+ defaultQualificationFeature?.max ??
+ 100,
+ ],
+ };
+ }
+ if (name === TENURE_FILTER_NAME) {
+ const defaultTenureFeatureName = getDefaultTenureFeatureName(features);
+ const defaultTenureFeature = defaultTenureFeatureName
+ ? features.find((feature) => feature.name === defaultTenureFeatureName)
+ : undefined;
+ if (!defaultTenureFeatureName) return prev;
+
+ return {
+ ...prev,
+ [createTenureFilterKey(defaultTenureFeatureName, tenureFilterIdRef.current++)]: [
+ defaultTenureFeature?.histogram?.min ?? defaultTenureFeature?.min ?? 0,
+ defaultTenureFeature?.histogram?.max ?? defaultTenureFeature?.max ?? 100,
+ ],
+ };
+ }
if (POI_FILTER_NAMES.includes(name as PoiFilterName)) {
const poiFilterName = name as PoiFilterName;
const defaultPoiFeatureName = getDefaultPoiFilterFeatureName(features, poiFilterName);
@@ -381,6 +488,23 @@ export function useFilters({
if (replaced) return normalizeFilters(next);
}
+ const crimeSeverityKeyId = getCrimeSeverityFilterKeyId(name);
+ if (crimeSeverityKeyId != null) {
+ let replaced = false;
+ const next: FeatureFilters = {};
+ for (const [existingName, existingValue] of Object.entries(prev)) {
+ if (getCrimeSeverityFilterKeyId(existingName) === crimeSeverityKeyId) {
+ if (!replaced) {
+ next[name] = value;
+ replaced = true;
+ }
+ continue;
+ }
+ next[existingName] = existingValue;
+ }
+ if (replaced) return normalizeFilters(next);
+ }
+
const ethnicityKeyId = getEthnicityFilterKeyId(name);
if (ethnicityKeyId != null) {
let replaced = false;
@@ -398,6 +522,40 @@ export function useFilters({
if (replaced) return normalizeFilters(next);
}
+ const qualificationKeyId = getQualificationFilterKeyId(name);
+ if (qualificationKeyId != null) {
+ let replaced = false;
+ const next: FeatureFilters = {};
+ for (const [existingName, existingValue] of Object.entries(prev)) {
+ if (getQualificationFilterKeyId(existingName) === qualificationKeyId) {
+ if (!replaced) {
+ next[name] = value;
+ replaced = true;
+ }
+ continue;
+ }
+ next[existingName] = existingValue;
+ }
+ if (replaced) return normalizeFilters(next);
+ }
+
+ const tenureKeyId = getTenureFilterKeyId(name);
+ if (tenureKeyId != null) {
+ let replaced = false;
+ const next: FeatureFilters = {};
+ for (const [existingName, existingValue] of Object.entries(prev)) {
+ if (getTenureFilterKeyId(existingName) === tenureKeyId) {
+ if (!replaced) {
+ next[name] = value;
+ replaced = true;
+ }
+ continue;
+ }
+ next[existingName] = existingValue;
+ }
+ if (replaced) return normalizeFilters(next);
+ }
+
const electionVoteShareKeyId = getElectionVoteShareFilterKeyId(name);
if (electionVoteShareKeyId != null) {
let replaced = false;
diff --git a/frontend/src/hooks/useHexagonSelection.ts b/frontend/src/hooks/useHexagonSelection.ts
index 9d90fd9..d73607b 100644
--- a/frontend/src/hooks/useHexagonSelection.ts
+++ b/frontend/src/hooks/useHexagonSelection.ts
@@ -10,7 +10,14 @@ import type {
HexagonPropertiesResponse,
HexagonStatsResponse,
} from '../types';
-import { buildFilterString, apiUrl, assertOk, logNonAbortError, authHeaders } from '../lib/api';
+import {
+ buildFilterString,
+ apiUrl,
+ assertOk,
+ logNonAbortError,
+ authHeaders,
+ fetchCrimeRecords,
+} from '../lib/api';
import { findOverlappingSelectableHexagon } from '../lib/h3-selection';
import { SMALLEST_VISIBLE_HEXAGON_RESOLUTION } from '../lib/consts';
import type { TravelTimeEntry } from './useTravelTime';
@@ -327,6 +334,23 @@ export function useHexagonSelection({
]
);
+ // Lazily fetch a page of individual crimes for the current selection. The
+ // records are an attribute of the area (filter-independent), so this carries
+ // no filter string — only the selection identity and share code.
+ const loadCrimeRecords = useCallback(
+ (offset: number) => {
+ if (!selectedHexagon) {
+ return Promise.resolve({ records: [], total: 0, offset: 0, truncated: false });
+ }
+ const selection =
+ selectedHexagon.type === 'postcode'
+ ? { postcode: selectedHexagon.id }
+ : { h3: selectedHexagon.id, resolution: selectedHexagon.resolution };
+ return fetchCrimeRecords(selection, offset, shareCode);
+ },
+ [selectedHexagon, shareCode]
+ );
+
const handleHexagonClick = useCallback(
(id: string, isPostcode = false, geometry?: PostcodeGeometry) => {
if (selectedHexagon?.id === id) {
@@ -806,6 +830,7 @@ export function useHexagonSelection({
handlePropertiesTabClick,
handleLoadMoreProperties,
handleCloseSelection,
+ loadCrimeRecords,
selectedPostcodeGeometry,
handleLocationSearch,
handleCurrentLocationSearch,
diff --git a/frontend/src/hooks/useLicense.ts b/frontend/src/hooks/useLicense.ts
index 6661718..3e6d626 100644
--- a/frontend/src/hooks/useLicense.ts
+++ b/frontend/src/hooks/useLicense.ts
@@ -1,8 +1,10 @@
import { useState, useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
import { apiUrl, authHeaders, assertOk } from '../lib/api';
import { trackEvent } from '../lib/analytics';
export function useLicense() {
+ const { t } = useTranslation();
const [checkingOut, setCheckingOut] = useState(false);
const [error, setError] = useState(null);
@@ -25,13 +27,13 @@ export function useLicense() {
window.location.href = data.url;
}
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Checkout failed';
+ const msg = err instanceof Error ? err.message : t('upgrade.checkoutFailed');
setError(msg);
throw err;
} finally {
setCheckingOut(false);
}
- }, []);
+ }, [t]);
return { startCheckout, checkingOut, error };
}
diff --git a/frontend/src/hooks/useListingLayers.ts b/frontend/src/hooks/useListingLayers.ts
index 75da1de..25c49ab 100644
--- a/frontend/src/hooks/useListingLayers.ts
+++ b/frontend/src/hooks/useListingLayers.ts
@@ -5,6 +5,13 @@ import Supercluster from 'supercluster';
import type { ActualListing } from '../types';
import { trackEvent } from '../lib/analytics';
+import { useClickedListings } from './useClickedListings';
+
+/** Default red pin vs. the violet used once a user has opened that listing. */
+const LISTING_PIN_FILL: [number, number, number, number] = [231, 76, 60, 240];
+const LISTING_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 240];
+const EXPANDED_PIN_FILL: [number, number, number, number] = [231, 76, 60, 245];
+const EXPANDED_PIN_VISITED_FILL: [number, number, number, number] = [139, 92, 246, 245];
const PRICE_LABEL_MIN_ZOOM = 14;
const ADDRESS_LABEL_MIN_ZOOM = 16;
@@ -113,6 +120,7 @@ function spiderfyPosition(
export function useListingLayers({ listings, zoom, isDark }: UseListingLayersProps) {
const [popupInfo, setPopupInfo] = useState(null);
const [selectedCluster, setSelectedCluster] = useState(null);
+ const { clickedUrls, markClicked } = useClickedListings();
useEffect(() => {
// Each refetch returns a fresh listings array and rebuilds the cluster index, so a
@@ -212,12 +220,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
[clearUnlockedPopup]
);
- const handleClick = useCallback((info: PickingInfo) => {
- const url = info.object?.listing_url;
- if (!url) return;
- trackEvent('Actual Listing Click', { url });
- window.open(url, '_blank', 'noopener,noreferrer');
- }, []);
+ const handleClick = useCallback(
+ (info: PickingInfo) => {
+ const url = info.object?.listing_url;
+ if (!url) return;
+ markClicked(url);
+ trackEvent('Actual Listing Click', { url });
+ window.open(url, '_blank', 'noopener,noreferrer');
+ },
+ [markClicked]
+ );
const handleHoverRef = useRef(handleHover);
handleHoverRef.current = handleHover;
@@ -244,12 +256,16 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
[clearUnlockedPopup]
);
- const handleExpandedClick = useCallback((info: PickingInfo) => {
- const url = info.object?.listing.listing_url;
- if (!url) return;
- trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' });
- window.open(url, '_blank', 'noopener,noreferrer');
- }, []);
+ const handleExpandedClick = useCallback(
+ (info: PickingInfo) => {
+ const url = info.object?.listing.listing_url;
+ if (!url) return;
+ markClicked(url);
+ trackEvent('Actual Listing Click', { url, source: 'cluster_expanded' });
+ window.open(url, '_blank', 'noopener,noreferrer');
+ },
+ [markClicked]
+ );
const handleExpandedHoverRef = useRef(handleExpandedHover);
handleExpandedHoverRef.current = handleExpandedHover;
@@ -345,7 +361,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
getPosition: (d) => [d.lon, d.lat],
getRadius: 7,
radiusUnits: 'pixels',
- getFillColor: [231, 76, 60, 240],
+ getFillColor: (d) =>
+ clickedUrls.has(d.listing_url) ? LISTING_PIN_VISITED_FILL : LISTING_PIN_FILL,
getLineColor: [255, 255, 255, 255],
getLineWidth: 1.5,
lineWidthUnits: 'pixels',
@@ -355,8 +372,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
highlightColor: [29, 228, 195, 220],
onHover: stableHover,
onClick: stableClick,
+ updateTriggers: { getFillColor: clickedUrls },
}),
- [visibleListings, stableHover, stableClick]
+ [visibleListings, clickedUrls, stableHover, stableClick]
);
const clusterShadowLayer = useMemo(
@@ -441,7 +459,8 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
getPosition: (d) => [d.lng, d.lat],
getRadius: 6,
radiusUnits: 'pixels',
- getFillColor: [231, 76, 60, 245],
+ getFillColor: (d) =>
+ clickedUrls.has(d.listing.listing_url) ? EXPANDED_PIN_VISITED_FILL : EXPANDED_PIN_FILL,
getLineColor: [255, 255, 255, 255],
getLineWidth: 1.5,
lineWidthUnits: 'pixels',
@@ -451,8 +470,9 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
highlightColor: [29, 228, 195, 220],
onHover: stableExpandedHover,
onClick: stableExpandedClick,
+ updateTriggers: { getFillColor: clickedUrls },
}),
- [expandedListings, stableExpandedHover, stableExpandedClick]
+ [expandedListings, clickedUrls, stableExpandedHover, stableExpandedClick]
);
const priceLabelLayer = useMemo(() => {
@@ -544,5 +564,11 @@ export function useListingLayers({ listings, zoom, isDark }: UseListingLayersPro
setSelectedCluster(null);
}, []);
- return { listingLayers, listingPopup: popupInfo, clearListingPopup };
+ return {
+ listingLayers,
+ listingPopup: popupInfo,
+ clearListingPopup,
+ clickedListingUrls: clickedUrls,
+ markListingClicked: markClicked,
+ };
}
diff --git a/frontend/src/hooks/useMapData.ts b/frontend/src/hooks/useMapData.ts
index 10b9f1f..fec2741 100644
--- a/frontend/src/hooks/useMapData.ts
+++ b/frontend/src/hooks/useMapData.ts
@@ -18,8 +18,11 @@ import {
} from '../lib/api';
import { getSchoolBackendFeatureName } from '../lib/school-filter';
import { getSpecificCrimeFeatureName } from '../lib/crime-filter';
+import { getCrimeSeverityFeatureName } from '../lib/crime-severity-filter';
import { getElectionVoteShareFeatureName } from '../lib/election-filter';
import { getEthnicityFeatureName } from '../lib/ethnicity-filter';
+import { getQualificationFeatureName } from '../lib/qualification-filter';
+import { getTenureFeatureName } from '../lib/tenure-filter';
import { getPoiDistanceFeatureName } from '../lib/poi-distance-filter';
import { POSTCODE_ZOOM_THRESHOLD } from '../lib/consts';
import { COLOR_RANGE_LOW_PERCENTILE, COLOR_RANGE_HIGH_PERCENTILE } from '../lib/consts';
@@ -119,8 +122,11 @@ export function useMapData({
(name: string) =>
getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ??
+ getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ??
+ getQualificationFeatureName(name) ??
+ getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ??
name,
[]
diff --git a/frontend/src/hooks/usePoiLayers.test.ts b/frontend/src/hooks/usePoiLayers.test.ts
index 472038c..c90fc37 100644
--- a/frontend/src/hooks/usePoiLayers.test.ts
+++ b/frontend/src/hooks/usePoiLayers.test.ts
@@ -1,9 +1,19 @@
import { act, renderHook } from '@testing-library/react';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
import type { POI } from '../types';
import { usePoiLayers } from './usePoiLayers';
+vi.mock('react-i18next', () => ({
+ useTranslation: () => ({
+ t: (key: string) => {
+ if (key === 'common.places') return 'places';
+ if (key === 'map.poi.zoomInToSeeDetails') return 'Zoom in to see details';
+ return key;
+ },
+ }),
+}));
+
const supermarket: POI = {
id: 'poi-1',
name: 'Market Hall',
diff --git a/frontend/src/hooks/usePoiLayers.ts b/frontend/src/hooks/usePoiLayers.ts
index fd1d791..766a0ff 100644
--- a/frontend/src/hooks/usePoiLayers.ts
+++ b/frontend/src/hooks/usePoiLayers.ts
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
import type { PickingInfo } from '@deck.gl/core';
import { IconLayer, ScatterplotLayer, TextLayer } from '@deck.gl/layers';
import Supercluster from 'supercluster';
@@ -65,6 +66,7 @@ function getPoiIconSize(poi: POI): number {
}
export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
+ const { t } = useTranslation();
const [popupInfo, setPopupInfo] = useState(null);
// Dismiss a lingering hover/cluster popup once the POIs behind it are gone — e.g.
@@ -108,8 +110,8 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
setPopupInfo({
x: info.x,
y: info.y,
- name: `${info.object.count} places`,
- category: 'Zoom in to see details',
+ name: `${info.object.count} ${t('common.places')}`,
+ category: t('map.poi.zoomInToSeeDetails'),
group: '',
emoji: '',
id: '',
@@ -119,7 +121,7 @@ export function usePoiLayers({ pois, zoom, isDark }: UsePoiLayersProps) {
} else {
setPopupInfo(null);
}
- }, []);
+ }, [t]);
const handleClusterHoverRef = useRef(handleClusterHover);
handleClusterHoverRef.current = handleClusterHover;
diff --git a/frontend/src/hooks/useRevealOnExpand.test.tsx b/frontend/src/hooks/useRevealOnExpand.test.tsx
new file mode 100644
index 0000000..1bd9c05
--- /dev/null
+++ b/frontend/src/hooks/useRevealOnExpand.test.tsx
@@ -0,0 +1,80 @@
+import { cleanup, fireEvent, render } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { useState } from 'react';
+
+import { useRevealOnExpand } from './useRevealOnExpand';
+
+function GroupList() {
+ const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
+ const [expanded, setExpanded] = useState(false);
+ return (
+
+
+
{
+ const willExpand = !expanded;
+ setExpanded(willExpand);
+ onToggle('g1', willExpand);
+ }}
+ >
+ toggle
+
+ {expanded &&
}
+
+
+
;
+
+ fireEvent.click(view.getByTestId('toggle'));
+
+ expect(scrollTo).toHaveBeenCalledWith({ top: 300 });
+ });
+
+ it('does not scroll when a group is collapsed', () => {
+ const view = render(;
+
+ fireEvent.click(view.getByTestId('toggle')); // expand
+ scrollTo.mockClear();
+ fireEvent.click(view.getByTestId('toggle')); // collapse
+
+ expect(scrollTo).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/src/hooks/useRevealOnExpand.ts b/frontend/src/hooks/useRevealOnExpand.ts
new file mode 100644
index 0000000..0c1ec0d
--- /dev/null
+++ b/frontend/src/hooks/useRevealOnExpand.ts
@@ -0,0 +1,97 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+/**
+ * Auto-scrolls a freshly expanded collapsible group into view inside a scroll
+ * container, so opening a group near the bottom reveals its content without
+ * manual scrolling. The (sticky) group header stays pinned at the top.
+ *
+ * Wiring:
+ * const { setContainer, registerGroup, onToggle } = useRevealOnExpand();
+ *
+ * {groups.map((g) => (
+ *
+ * {
+ * const willExpand = !isExpanded(g.name);
+ * toggle(g.name);
+ * onToggle(g.name, willExpand);
+ * }}
+ * />
+ * {isExpanded(g.name) && }
+ *
+ * ))}
+ *
() {
+ const containerRef = useRef(null);
+ const groupRefs = useRef>(new Map());
+ const groupRefCallbacks = useRef void>>(new Map());
+ // A fresh object each toggle so re-expanding the same group still re-triggers.
+ const [groupToReveal, setGroupToReveal] = useState<{ name: string } | null>(null);
+
+ const setContainer = useCallback((node: T | null) => {
+ containerRef.current = node;
+ }, []);
+
+ // Returns a stable ref callback per group name so React doesn't churn the map
+ // (cleanup + re-register) on every render.
+ const registerGroup = useCallback((name: string) => {
+ const existing = groupRefCallbacks.current.get(name);
+ if (existing) return existing;
+ const cb = (node: T | null) => {
+ if (node) groupRefs.current.set(name, node);
+ else groupRefs.current.delete(name);
+ };
+ groupRefCallbacks.current.set(name, cb);
+ return cb;
+ }, []);
+
+ // Call from a group's toggle handler: reveals the group when it is being
+ // expanded, and cancels any pending reveal when it is being collapsed.
+ const onToggle = useCallback((name: string, willExpand: boolean) => {
+ setGroupToReveal(willExpand ? { name } : null);
+ }, []);
+
+ useEffect(() => {
+ if (!groupToReveal) return;
+ const el = groupRefs.current.get(groupToReveal.name);
+ const container = containerRef.current;
+ if (!el || !container) return;
+
+ // Scroll down just enough to bring the bottom of the opened group into view.
+ // For groups taller than the pane this scrolls past the header, but it stays
+ // visible via its sticky positioning. Never scroll up (group already in view).
+ const alignBottom = () => {
+ const delta = el.getBoundingClientRect().bottom - container.getBoundingClientRect().bottom;
+ if (delta <= 1) return;
+ container.scrollTo({ top: container.scrollTop + delta });
+ };
+
+ alignBottom();
+
+ // The group's charts and async data (e.g. nearby stations) can grow its
+ // height after this first pass, so keep the bottom pinned as it settles.
+ // Stop once the user scrolls or after a short grace period so we never fight
+ // a deliberate scroll.
+ let timer = 0;
+ const observer = new ResizeObserver(alignBottom);
+ const stop = () => {
+ observer.disconnect();
+ container.removeEventListener('wheel', stop);
+ container.removeEventListener('touchmove', stop);
+ window.clearTimeout(timer);
+ };
+ observer.observe(el);
+ container.addEventListener('wheel', stop, { passive: true });
+ container.addEventListener('touchmove', stop, { passive: true });
+ timer = window.setTimeout(stop, 1500);
+
+ return stop;
+ }, [groupToReveal]);
+
+ return { setContainer, registerGroup, onToggle };
+}
diff --git a/frontend/src/hooks/useSavedSearches.ts b/frontend/src/hooks/useSavedSearches.ts
index c69d5bd..f37024e 100644
--- a/frontend/src/hooks/useSavedSearches.ts
+++ b/frontend/src/hooks/useSavedSearches.ts
@@ -1,4 +1,5 @@
import { useState, useCallback, useRef, useEffect } from 'react';
+import { useTranslation } from 'react-i18next';
import pb from '../lib/pocketbase';
import { apiUrl, authHeaders } from '../lib/api';
import { trackEvent } from '../lib/analytics';
@@ -24,6 +25,7 @@ function nextPollDelay(attempt: number): number {
}
export function useSavedSearches(userId: string | null) {
+ const { t } = useTranslation();
const [searches, setSearches] = useState([]);
const [loading, setLoading] = useState(false);
const [saving, setSaving] = useState(false);
@@ -127,11 +129,11 @@ export function useSavedSearches(userId: string | null) {
stopPolling();
}
} catch (err) {
- setError(err instanceof Error ? err.message : 'Failed to load searches');
+ setError(err instanceof Error ? err.message : t('savedPage.loadFailed'));
} finally {
setLoading(false);
}
- }, [userId, fetchRecords, startPolling, stopPolling]);
+ }, [userId, fetchRecords, startPolling, stopPolling, t]);
const saveSearch = useCallback(
async (name: string, paramsOverride?: string) => {
@@ -169,14 +171,14 @@ export function useSavedSearches(userId: string | null) {
console.warn('Background screenshot failed:', err);
});
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Failed to save search';
+ const msg = err instanceof Error ? err.message : t('savedPage.saveFailed');
setError(msg);
throw err;
} finally {
setSaving(false);
}
},
- [userId, fetchSearches]
+ [userId, fetchSearches, t]
);
const deleteSearch = useCallback(async (id: string) => {
@@ -186,27 +188,27 @@ export function useSavedSearches(userId: string | null) {
trackEvent('Search Delete');
setSearches((prev) => prev.filter((s) => s.id !== id));
} catch (err) {
- setError(err instanceof Error ? err.message : 'Failed to delete search');
+ setError(err instanceof Error ? err.message : t('savedPage.deleteFailed'));
}
- }, []);
+ }, [t]);
const updateSearchNotes = useCallback(async (id: string, notes: string) => {
try {
await pb.collection('saved_searches').update(id, { notes });
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, notes } : s)));
} catch (err) {
- setError(err instanceof Error ? err.message : 'Failed to update notes');
+ setError(err instanceof Error ? err.message : t('savedPage.updateNotesFailed'));
}
- }, []);
+ }, [t]);
const updateSearchName = useCallback(async (id: string, name: string) => {
try {
await pb.collection('saved_searches').update(id, { name });
setSearches((prev) => prev.map((s) => (s.id === id ? { ...s, name } : s)));
} catch (err) {
- setError(err instanceof Error ? err.message : 'Failed to update name');
+ setError(err instanceof Error ? err.message : t('savedPage.updateNameFailed'));
}
- }, []);
+ }, [t]);
const updateSearchParams = useCallback(
async (id: string, params: string) => {
@@ -238,14 +240,14 @@ export function useSavedSearches(userId: string | null) {
console.warn('Background screenshot failed:', err);
});
} catch (err) {
- const msg = err instanceof Error ? err.message : 'Failed to update search';
+ const msg = err instanceof Error ? err.message : t('savedPage.updateFailed');
setError(msg);
throw err;
} finally {
setSaving(false);
}
},
- [userId, fetchSearches]
+ [userId, fetchSearches, t]
);
return {
diff --git a/frontend/src/hooks/useTutorial.ts b/frontend/src/hooks/useTutorial.ts
index 423094f..a5b4b4a 100644
--- a/frontend/src/hooks/useTutorial.ts
+++ b/frontend/src/hooks/useTutorial.ts
@@ -50,6 +50,13 @@ export function useTutorial(initialLoading: boolean, isMobile: boolean, blocked
placement: 'left' as const,
skipBeacon: true,
},
+ {
+ target: '[data-tutorial="overlays-button"]',
+ title: t('tutorial.step7Title'),
+ content: t('tutorial.step7Content'),
+ placement: 'left' as const,
+ skipBeacon: true,
+ },
],
[t]
);
diff --git a/frontend/src/index.tsx b/frontend/src/index.tsx
index 38abb5b..8190c55 100644
--- a/frontend/src/index.tsx
+++ b/frontend/src/index.tsx
@@ -1,4 +1,5 @@
import { createRoot } from 'react-dom/client';
+import { useTranslation } from 'react-i18next';
import App from './App';
import { i18nReady } from './i18n';
import { BugsinkErrorBoundary, initBugsink } from './lib/bugsink';
@@ -14,12 +15,17 @@ if (!container) {
const root = container;
function AppErrorFallback() {
+ const { t } = useTranslation();
return (
-
Something went wrong
+
+ {t('errors.appCrashTitle', { defaultValue: 'Something went wrong' })}
+
- Refresh the page to try again.
+ {t('errors.appCrashBody', {
+ defaultValue: 'Refresh the page to try again.',
+ })}
diff --git a/frontend/src/lib/api.test.ts b/frontend/src/lib/api.test.ts
index 75019f2..f9c0d9d 100644
--- a/frontend/src/lib/api.test.ts
+++ b/frontend/src/lib/api.test.ts
@@ -6,6 +6,8 @@ import { createSchoolFilterKey } from './school-filter';
import { createSpecificCrimeFilterKey } from './crime-filter';
import { createElectionVoteShareFilterKey } from './election-filter';
import { createEthnicityFilterKey } from './ethnicity-filter';
+import { createQualificationFilterKey } from './qualification-filter';
+import { createTenureFilterKey } from './tenure-filter';
import {
POI_COUNT_2KM_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME,
@@ -98,19 +100,19 @@ describe('api utilities', () => {
it('serializes specific crime filters using their selected backend crime feature', () => {
const features: FeatureMeta[] = [
- { name: 'Burglary (avg/yr)', type: 'numeric', min: 0, max: 20 },
- { name: 'Vehicle crime (avg/yr)', type: 'numeric', min: 0, max: 30 },
+ { name: 'Burglary (/yr, 7y)', type: 'numeric', min: 0, max: 20 },
+ { name: 'Vehicle crime (/yr, 7y)', type: 'numeric', min: 0, max: 30 },
];
expect(
buildFilterString(
{
- [createSpecificCrimeFilterKey('Burglary (avg/yr)', 1)]: [0, 5],
- [createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2)]: [1, 10],
+ [createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1)]: [0, 5],
+ [createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2)]: [1, 10],
},
features
)
- ).toBe('Burglary (avg/yr):0:5;;Vehicle crime (avg/yr):1:10');
+ ).toBe('Burglary (/yr, 7y):0:5;;Vehicle crime (/yr, 7y):1:10');
});
it('serializes election vote-share filters using their selected backend party feature', () => {
@@ -144,6 +146,40 @@ describe('api utilities', () => {
).toBe('% White:20:80');
});
+ it('serializes qualification filters using their selected backend band feature', () => {
+ const features: FeatureMeta[] = [
+ { name: '% Degree or higher', type: 'numeric', min: 0, max: 100 },
+ { name: '% No qualifications', type: 'numeric', min: 0, max: 100 },
+ ];
+
+ expect(
+ buildFilterString(
+ {
+ [createQualificationFilterKey('% Degree or higher', 1)]: [20, 60],
+ [createQualificationFilterKey('% No qualifications', 2)]: [0, 25],
+ },
+ features
+ )
+ ).toBe('% Degree or higher:20:60;;% No qualifications:0:25');
+ });
+
+ it('serializes tenure filters using their selected backend band feature', () => {
+ const features: FeatureMeta[] = [
+ { name: '% Owner occupied', type: 'numeric', min: 0, max: 100 },
+ { name: '% Private rent', type: 'numeric', min: 0, max: 100 },
+ ];
+
+ expect(
+ buildFilterString(
+ {
+ [createTenureFilterKey('% Owner occupied', 1)]: [20, 60],
+ [createTenureFilterKey('% Private rent', 2)]: [0, 25],
+ },
+ features
+ )
+ ).toBe('% Owner occupied:20:60;;% Private rent:0:25');
+ });
+
it('serializes amenity distance filters using their selected backend feature', () => {
const features: FeatureMeta[] = [
{ name: 'Distance to nearest amenity (Park) (km)', type: 'numeric', min: 0, max: 2 },
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index be62d98..e77a25b 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -1,10 +1,13 @@
-import type { FeatureMeta, FeatureFilters } from '../types';
+import type { FeatureMeta, FeatureFilters, CrimeRecordsResponse } from '../types';
import { INITIAL_RETRY_MS, MAX_RETRY_MS } from './consts';
import pb from './pocketbase';
import { getSchoolBackendFeatureName } from './school-filter';
import { getSpecificCrimeFeatureName } from './crime-filter';
+import { getCrimeSeverityFeatureName } from './crime-severity-filter';
import { getElectionVoteShareFeatureName } from './election-filter';
import { getEthnicityFeatureName } from './ethnicity-filter';
+import { getQualificationFeatureName } from './qualification-filter';
+import { getTenureFeatureName } from './tenure-filter';
import { getPoiDistanceFeatureName } from './poi-distance-filter';
const SCREENSHOT_LANGUAGES = new Set(['en', 'fr', 'de', 'zh', 'hi', 'hu']);
@@ -61,6 +64,7 @@ const DEMO_GATED_ENDPOINTS = new Set([
'postcode-stats',
'postcode-properties',
'hexagon-properties',
+ 'crime-records',
'journey',
'actual-listings',
'developments',
@@ -186,6 +190,26 @@ export async function shortenUrl(params: string, language?: string): Promise {
+ const params = new URLSearchParams({ offset: offset.toString() });
+ if ('postcode' in selection) {
+ params.set('postcode', selection.postcode);
+ } else {
+ params.set('h3', selection.h3);
+ params.set('resolution', selection.resolution.toString());
+ }
+ if (shareCode) params.set('share', shareCode);
+ const res = await fetch(apiUrl('crime-records', params), authHeaders());
+ assertOk(res, 'crime-records');
+ return res.json();
+}
+
export function buildFilterString(
filters: FeatureFilters,
features: FeatureMeta[],
@@ -200,8 +224,11 @@ export function buildFilterString(
const backendName =
getSchoolBackendFeatureName(name) ??
getSpecificCrimeFeatureName(name) ??
+ getCrimeSeverityFeatureName(name) ??
getElectionVoteShareFeatureName(name) ??
getEthnicityFeatureName(name) ??
+ getQualificationFeatureName(name) ??
+ getTenureFeatureName(name) ??
getPoiDistanceFeatureName(name) ??
name;
const prev = merged.get(backendName);
diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts
index e01f565..5846f8f 100644
--- a/frontend/src/lib/consts.ts
+++ b/frontend/src/lib/consts.ts
@@ -56,7 +56,7 @@ export const SMALLEST_VISIBLE_HEXAGON_RESOLUTION = Math.max(
// past the finest hexagon level so detail appears while still relatively
// zoomed out. (Each overlay additionally can't render below its own tile-data
// floor, OVERLAY_MIN_ZOOM, regardless of this limit.)
-export const POSTCODE_ZOOM_THRESHOLD = 14;
+export const POSTCODE_ZOOM_THRESHOLD = 13.5;
export const POSTCODE_SEARCH_ZOOM = 16;
export const FEATURE_GRADIENT: { t: number; color: [number, number, number] }[] = [
@@ -264,7 +264,7 @@ export const STACKED_GROUPS: Record<
label: string;
/** If set, use this feature's stats for the total and info popup. Otherwise sum components. */
feature?: string;
- /** If set, display this feature's mean as the primary value (e.g. per-1k rate) instead of the absolute total. */
+ /** If set, display this feature's mean as the primary value (e.g. a percentage) instead of the absolute total. */
rateFeature?: string;
/** Suffix shown after the total value (e.g. "avg/yr") */
unit?: string;
@@ -275,30 +275,30 @@ export const STACKED_GROUPS: Record<
Crime: [
{
label: 'Serious crime',
- feature: 'Serious crime (avg/yr)',
+ feature: 'Serious crime (/yr, 7y)',
unit: '',
components: [
- 'Violence and sexual offences (avg/yr)',
- 'Robbery (avg/yr)',
- 'Burglary (avg/yr)',
- 'Possession of weapons (avg/yr)',
+ 'Violence and sexual offences (/yr, 7y)',
+ 'Robbery (/yr, 7y)',
+ 'Burglary (/yr, 7y)',
+ 'Possession of weapons (/yr, 7y)',
],
},
{
label: 'Minor crime',
- feature: 'Minor crime (avg/yr)',
+ feature: 'Minor crime (/yr, 7y)',
unit: '',
components: [
- 'Anti-social behaviour (avg/yr)',
- 'Criminal damage and arson (avg/yr)',
- 'Shoplifting (avg/yr)',
- 'Bicycle theft (avg/yr)',
- 'Theft from the person (avg/yr)',
- 'Other theft (avg/yr)',
- 'Vehicle crime (avg/yr)',
- 'Public order (avg/yr)',
- 'Drugs (avg/yr)',
- 'Other crime (avg/yr)',
+ 'Anti-social behaviour (/yr, 7y)',
+ 'Criminal damage and arson (/yr, 7y)',
+ 'Shoplifting (/yr, 7y)',
+ 'Bicycle theft (/yr, 7y)',
+ 'Theft from the person (/yr, 7y)',
+ 'Other theft (/yr, 7y)',
+ 'Vehicle crime (/yr, 7y)',
+ 'Public order (/yr, 7y)',
+ 'Drugs (/yr, 7y)',
+ 'Other crime (/yr, 7y)',
],
},
],
@@ -493,20 +493,20 @@ export function getEnumValueColor(
/** Explicit colors for stacked bar segments. */
export const STACKED_SEGMENT_COLORS: Record = {
- 'Violence and sexual offences (avg/yr)': '#ef4444',
- 'Robbery (avg/yr)': '#f97316',
- 'Burglary (avg/yr)': '#eab308',
- 'Possession of weapons (avg/yr)': '#8b5cf6',
- 'Anti-social behaviour (avg/yr)': '#14b8a6',
- 'Criminal damage and arson (avg/yr)': '#f97316',
- 'Shoplifting (avg/yr)': '#ec4899',
- 'Bicycle theft (avg/yr)': '#22c55e',
- 'Theft from the person (avg/yr)': '#d946ef',
- 'Other theft (avg/yr)': '#06b6d4',
- 'Vehicle crime (avg/yr)': '#3b82f6',
- 'Public order (avg/yr)': '#8b5cf6',
- 'Drugs (avg/yr)': '#22c55e',
- 'Other crime (avg/yr)': '#6b7280',
+ 'Violence and sexual offences (/yr, 7y)': '#ef4444',
+ 'Robbery (/yr, 7y)': '#f97316',
+ 'Burglary (/yr, 7y)': '#eab308',
+ 'Possession of weapons (/yr, 7y)': '#8b5cf6',
+ 'Anti-social behaviour (/yr, 7y)': '#14b8a6',
+ 'Criminal damage and arson (/yr, 7y)': '#f97316',
+ 'Shoplifting (/yr, 7y)': '#ec4899',
+ 'Bicycle theft (/yr, 7y)': '#22c55e',
+ 'Theft from the person (/yr, 7y)': '#d946ef',
+ 'Other theft (/yr, 7y)': '#06b6d4',
+ 'Vehicle crime (/yr, 7y)': '#3b82f6',
+ 'Public order (/yr, 7y)': '#8b5cf6',
+ 'Drugs (/yr, 7y)': '#22c55e',
+ 'Other crime (/yr, 7y)': '#6b7280',
'% White': '#3b82f6',
'% South Asian': '#f97316',
'% East Asian': '#eab308',
diff --git a/frontend/src/lib/crime-filter.test.ts b/frontend/src/lib/crime-filter.test.ts
new file mode 100644
index 0000000..0775565
--- /dev/null
+++ b/frontend/src/lib/crime-filter.test.ts
@@ -0,0 +1,79 @@
+import { describe, expect, it } from 'vitest';
+import type { FeatureMeta } from '../types';
+import {
+ SPECIFIC_CRIME_FEATURE_NAMES,
+ createSpecificCrimeFilterKey,
+ getDefaultSpecificCrimeFeatureName,
+ getSpecificCrimeFeatureName,
+ getSpecificCrimeFilterKeyId,
+ getSpecificCrimeType,
+ getSpecificCrimeWindow,
+ isSpecificCrimeFeatureName,
+ isSpecificCrimeFilterName,
+ normalizeSpecificCrimeFilters,
+ specificCrimeFeatureName,
+ withSpecificCrimeWindow,
+} from './crime-filter';
+
+const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
+
+describe('specific-crime window helpers', () => {
+ it('recognizes both the 7-year and 2-year rate features', () => {
+ expect(isSpecificCrimeFeatureName('Burglary (/yr, 7y)')).toBe(true);
+ expect(isSpecificCrimeFeatureName('Burglary (/yr, 2y)')).toBe(true);
+ expect(isSpecificCrimeFeatureName('Burglary (avg/yr)')).toBe(false);
+ // The "Serious crime" / "Minor crime" aggregates stay separate sliders.
+ expect(isSpecificCrimeFeatureName('Serious crime (/yr, 7y)')).toBe(false);
+ });
+
+ it('extracts the window and bare type', () => {
+ expect(getSpecificCrimeWindow('Burglary (/yr, 2y)')).toBe('2y');
+ expect(getSpecificCrimeWindow('Burglary (/yr, 7y)')).toBe('7y');
+ expect(getSpecificCrimeWindow('Burglary')).toBeNull();
+ expect(getSpecificCrimeType('Violence and sexual offences (/yr, 2y)')).toBe(
+ 'Violence and sexual offences'
+ );
+ expect(getSpecificCrimeType('not a crime')).toBeNull();
+ });
+
+ it('swaps a feature name between windows and round-trips', () => {
+ expect(withSpecificCrimeWindow('Burglary (/yr, 7y)', '2y')).toBe(
+ 'Burglary (/yr, 2y)'
+ );
+ expect(withSpecificCrimeWindow('Burglary (/yr, 2y)', '7y')).toBe(
+ 'Burglary (/yr, 7y)'
+ );
+ // Unrecognized names pass through untouched.
+ expect(withSpecificCrimeWindow('Burglary', '2y')).toBe('Burglary');
+ expect(specificCrimeFeatureName('Drugs', '2y')).toBe('Drugs (/yr, 2y)');
+ });
+
+ it('defaults to the 7-year window and enumerates 7-year dropdown names', () => {
+ const features = [numeric('Burglary (/yr, 7y)'), numeric('Burglary (/yr, 2y)')];
+ expect(getDefaultSpecificCrimeFeatureName(features)).toBe('Burglary (/yr, 7y)');
+ expect(SPECIFIC_CRIME_FEATURE_NAMES.every((n) => n.endsWith('(/yr, 7y)'))).toBe(true);
+ });
+});
+
+describe('specific-crime filter keys', () => {
+ it('resolves a 2-year filter key back to its backend column', () => {
+ const key = createSpecificCrimeFilterKey('Burglary (/yr, 2y)', 3);
+ expect(isSpecificCrimeFilterName(key)).toBe(true);
+ expect(getSpecificCrimeFilterKeyId(key)).toBe('3');
+ expect(getSpecificCrimeFeatureName(key)).toBe('Burglary (/yr, 2y)');
+ });
+
+ it('folds a bare 2-year crime feature into a multi-instance key', () => {
+ const normalized = normalizeSpecificCrimeFilters({
+ 'Burglary (/yr, 2y)': [1, 5],
+ 'Median price (£)': [0, 100],
+ });
+ const keys = Object.keys(normalized);
+ expect(keys.some((k) => isSpecificCrimeFilterName(k))).toBe(true);
+ const crimeKey = keys.find((k) => isSpecificCrimeFilterName(k))!;
+ expect(getSpecificCrimeFeatureName(crimeKey)).toBe('Burglary (/yr, 2y)');
+ expect(normalized[crimeKey]).toEqual([1, 5]);
+ // Non-crime filters are left untouched.
+ expect(normalized['Median price (£)']).toEqual([0, 100]);
+ });
+});
diff --git a/frontend/src/lib/crime-filter.ts b/frontend/src/lib/crime-filter.ts
index 9e6ad0a..a5145b8 100644
--- a/frontend/src/lib/crime-filter.ts
+++ b/frontend/src/lib/crime-filter.ts
@@ -1,26 +1,74 @@
import type { FeatureFilters, FeatureMeta } from '../types';
+import type { VariantFilterConfig } from './variant-filter';
export const SPECIFIC_CRIMES_FILTER_NAME = 'Specific crimes';
export const SPECIFIC_CRIMES_FILTER_KEY_PREFIX = `${SPECIFIC_CRIMES_FILTER_NAME}:`;
-export const SPECIFIC_CRIME_FEATURE_NAMES = [
- 'Violence and sexual offences (avg/yr)',
- 'Burglary (avg/yr)',
- 'Robbery (avg/yr)',
- 'Vehicle crime (avg/yr)',
- 'Anti-social behaviour (avg/yr)',
- 'Criminal damage and arson (avg/yr)',
- 'Other theft (avg/yr)',
- 'Theft from the person (avg/yr)',
- 'Shoplifting (avg/yr)',
- 'Bicycle theft (avg/yr)',
- 'Drugs (avg/yr)',
- 'Possession of weapons (avg/yr)',
- 'Public order (avg/yr)',
- 'Other crime (avg/yr)',
+// The "pick one crime type" filter selects a single street-level category, with
+// a toggle to measure it over the 7-year or the recent 2-year window. The 7-year
+// window is the default/primary one (also headlined in the area pane).
+export const SPECIFIC_CRIME_TYPES = [
+ 'Violence and sexual offences',
+ 'Burglary',
+ 'Robbery',
+ 'Vehicle crime',
+ 'Anti-social behaviour',
+ 'Criminal damage and arson',
+ 'Other theft',
+ 'Theft from the person',
+ 'Shoplifting',
+ 'Bicycle theft',
+ 'Drugs',
+ 'Possession of weapons',
+ 'Public order',
+ 'Other crime',
] as const;
-const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set(SPECIFIC_CRIME_FEATURE_NAMES);
+export const SPECIFIC_CRIME_WINDOW_DEFAULT = '7y';
+export const SPECIFIC_CRIME_WINDOWS = ['7y', '2y'] as const;
+export type SpecificCrimeWindow = (typeof SPECIFIC_CRIME_WINDOWS)[number];
+
+/** The backend rate-feature column for a crime type in a given window. */
+export function specificCrimeFeatureName(type: string, window: SpecificCrimeWindow): string {
+ return `${type} (/yr, ${window})`;
+}
+
+// Canonical dropdown enumeration: one feature name per crime type, in the
+// default (7-year) window. The window toggle re-points these to the chosen window.
+export const SPECIFIC_CRIME_FEATURE_NAMES = SPECIFIC_CRIME_TYPES.map((type) =>
+ specificCrimeFeatureName(type, SPECIFIC_CRIME_WINDOW_DEFAULT)
+);
+
+// Every recognized specific-crime feature, across all windows. Used to detect /
+// resolve a filter key's backend column for either window.
+const SPECIFIC_CRIME_FEATURE_NAME_SET = new Set(
+ SPECIFIC_CRIME_TYPES.flatMap((type) =>
+ SPECIFIC_CRIME_WINDOWS.map((window) => specificCrimeFeatureName(type, window))
+ )
+);
+
+const SPECIFIC_CRIME_WINDOW_RE = / \(\/yr, (7y|2y)\)$/;
+
+/** The window suffix of a specific-crime feature name (e.g. "2y"), or null. */
+export function getSpecificCrimeWindow(featureName: string): SpecificCrimeWindow | null {
+ const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
+ return match ? (match[1] as SpecificCrimeWindow) : null;
+}
+
+/** The bare crime type of a specific-crime feature name (e.g. "Burglary"), or null. */
+export function getSpecificCrimeType(featureName: string): string | null {
+ const match = featureName.match(SPECIFIC_CRIME_WINDOW_RE);
+ return match ? featureName.slice(0, match.index) : null;
+}
+
+/** The same crime type's feature name in a different window (no-op if unrecognized). */
+export function withSpecificCrimeWindow(
+ featureName: string,
+ window: SpecificCrimeWindow
+): string {
+ const type = getSpecificCrimeType(featureName);
+ return type ? specificCrimeFeatureName(type, window) : featureName;
+}
export function isSpecificCrimeFeatureName(name: string): boolean {
return SPECIFIC_CRIME_FEATURE_NAME_SET.has(name);
@@ -101,7 +149,7 @@ export function getSpecificCrimeFilterMeta(features: FeatureMeta[]): FeatureMeta
description:
'Violence, burglary, robbery, drugs, shoplifting, vehicle crime, anti-social behaviour, public order, theft, and other crime types',
detail:
- 'Filter by one street-level crime category at a time using an area-normalised crime density near each postcode (not a count of incidents per year).',
+ 'Filter by one street-level crime category at a time, as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
source: 'crime',
suffix: '',
};
@@ -115,3 +163,26 @@ export function clampSpecificCrimeRange(
const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
}
+
+export const SPECIFIC_CRIME_VARIANT_CONFIG: VariantFilterConfig = {
+ filterName: SPECIFIC_CRIMES_FILTER_NAME,
+ featureNames: SPECIFIC_CRIME_FEATURE_NAMES,
+ dropdownLabelKey: 'filters.crimeType',
+ getFilterMeta: getSpecificCrimeFilterMeta,
+ getDefaultFeatureName: getDefaultSpecificCrimeFeatureName,
+ getFeatureName: getSpecificCrimeFeatureName,
+ replaceFilterKeySelection: replaceSpecificCrimeFilterKeySelection,
+ clampRange: clampSpecificCrimeRange,
+ // Dropdown shows the bare crime type; the window toggle carries the 7y/2y suffix.
+ getOptionLabelSource: (name) => getSpecificCrimeType(name) ?? name,
+ window: {
+ labelKey: 'filters.crimeWindow',
+ options: [
+ { id: '7y', labelKey: 'filters.crimeWindow7y' },
+ { id: '2y', labelKey: 'filters.crimeWindow2y' },
+ ],
+ getWindow: getSpecificCrimeWindow,
+ withWindow: (name, windowId) =>
+ withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
+ },
+};
diff --git a/frontend/src/lib/crime-severity-filter.test.ts b/frontend/src/lib/crime-severity-filter.test.ts
new file mode 100644
index 0000000..4ef86db
--- /dev/null
+++ b/frontend/src/lib/crime-severity-filter.test.ts
@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest';
+import type { FeatureMeta } from '../types';
+import {
+ CRIME_SEVERITY_FILTER_NAMES,
+ createCrimeSeverityFilterKey,
+ getCrimeSeverityFeatureName,
+ getCrimeSeverityFilterKeyId,
+ getCrimeSeverityFilterName,
+ getCrimeSeverityVariantConfig,
+ getDefaultCrimeSeverityFeatureName,
+ isCrimeSeverityFeatureName,
+ isCrimeSeverityFilterName,
+ normalizeCrimeSeverityFilters,
+} from './crime-severity-filter';
+
+const numeric = (name: string): FeatureMeta => ({ name, type: 'numeric', min: 0, max: 10 });
+
+describe('crime-severity feature recognition', () => {
+ it('recognizes Serious/Minor crime in both windows only', () => {
+ expect(isCrimeSeverityFeatureName('Serious crime (/yr, 7y)')).toBe(true);
+ expect(isCrimeSeverityFeatureName('Serious crime (/yr, 2y)')).toBe(true);
+ expect(isCrimeSeverityFeatureName('Minor crime (/yr, 2y)')).toBe(true);
+ // Detailed categories belong to "Specific crimes", not severity.
+ expect(isCrimeSeverityFeatureName('Burglary (/yr, 7y)')).toBe(false);
+ // Bare names without a window suffix are not feature columns.
+ expect(isCrimeSeverityFeatureName('Serious crime')).toBe(false);
+ });
+
+ it('maps a feature or key back to its severity filter name', () => {
+ expect(getCrimeSeverityFilterName('Serious crime (/yr, 2y)')).toBe('Serious crime');
+ expect(getCrimeSeverityFilterName('Minor crime (/yr, 7y)')).toBe('Minor crime');
+ const key = createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0);
+ expect(getCrimeSeverityFilterName(key)).toBe('Serious crime');
+ expect(getCrimeSeverityFilterName('Burglary (/yr, 7y)')).toBeNull();
+ });
+});
+
+describe('crime-severity filter keys', () => {
+ it('resolves a 2-year key back to its backend column and id', () => {
+ const key = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 4);
+ expect(isCrimeSeverityFilterName(key)).toBe(true);
+ expect(getCrimeSeverityFilterKeyId(key)).toBe('4');
+ expect(getCrimeSeverityFeatureName(key)).toBe('Minor crime (/yr, 2y)');
+ });
+
+ it('defaults to the 7-year window when present', () => {
+ const features = [
+ numeric('Serious crime (/yr, 7y)'),
+ numeric('Serious crime (/yr, 2y)'),
+ ];
+ expect(getDefaultCrimeSeverityFeatureName(features, 'Serious crime')).toBe(
+ 'Serious crime (/yr, 7y)'
+ );
+ expect(getDefaultCrimeSeverityFeatureName(features, 'Minor crime')).toBeNull();
+ });
+
+ it('folds bare severity features but leaves specific crimes and others alone', () => {
+ const normalized = normalizeCrimeSeverityFilters({
+ 'Serious crime (/yr, 2y)': [1, 5],
+ 'Burglary (/yr, 7y)': [0, 3],
+ 'Median price (£)': [0, 100],
+ });
+ const keys = Object.keys(normalized);
+ const severityKey = keys.find((k) => isCrimeSeverityFilterName(k))!;
+ expect(getCrimeSeverityFeatureName(severityKey)).toBe('Serious crime (/yr, 2y)');
+ expect(normalized[severityKey]).toEqual([1, 5]);
+ // Specific-crime + plain features pass through untouched.
+ expect(normalized['Burglary (/yr, 7y)']).toEqual([0, 3]);
+ expect(normalized['Median price (£)']).toEqual([0, 100]);
+ });
+});
+
+describe('crime-severity variant config', () => {
+ it('exposes a single variant per severity with a 7y/2y window toggle', () => {
+ for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
+ const config = getCrimeSeverityVariantConfig(filterName);
+ expect(config.filterName).toBe(filterName);
+ expect(config.featureNames).toEqual([`${filterName} (/yr, 7y)`]);
+ expect(config.window?.options.map((o) => o.id)).toEqual(['7y', '2y']);
+ // Switching window re-points to the same severity, other window.
+ const sevenYear = `${filterName} (/yr, 7y)`;
+ expect(config.window?.withWindow(sevenYear, '2y')).toBe(`${filterName} (/yr, 2y)`);
+ }
+ });
+});
diff --git a/frontend/src/lib/crime-severity-filter.ts b/frontend/src/lib/crime-severity-filter.ts
new file mode 100644
index 0000000..06835d6
--- /dev/null
+++ b/frontend/src/lib/crime-severity-filter.ts
@@ -0,0 +1,193 @@
+import type { FeatureFilters, FeatureMeta } from '../types';
+import type { VariantFilterConfig } from './variant-filter';
+import {
+ SPECIFIC_CRIME_WINDOW_DEFAULT,
+ clampSpecificCrimeRange,
+ getSpecificCrimeType,
+ getSpecificCrimeWindow,
+ specificCrimeFeatureName,
+ withSpecificCrimeWindow,
+ type SpecificCrimeWindow,
+} from './crime-filter';
+
+// "Serious crime" and "Minor crime" are aggregate severity rollups (they overlap
+// the 14 detailed categories, so they must stay OUT of the "Specific crimes"
+// dropdown). Each is a single feature with a 7-year/2-year window toggle — the
+// same folding the specific-crimes card has, but with no variant dropdown (one
+// variant). Modeled on the multi-filter-name POI pattern: one lib, two filter
+// names, distinguished by their key prefix / bare crime type.
+
+export const SERIOUS_CRIME_FILTER_NAME = 'Serious crime';
+export const MINOR_CRIME_FILTER_NAME = 'Minor crime';
+
+export const CRIME_SEVERITY_FILTER_NAMES = [
+ SERIOUS_CRIME_FILTER_NAME,
+ MINOR_CRIME_FILTER_NAME,
+] as const;
+export type CrimeSeverityFilterName = (typeof CRIME_SEVERITY_FILTER_NAMES)[number];
+
+const CRIME_SEVERITY_TYPE_SET = new Set(CRIME_SEVERITY_FILTER_NAMES);
+
+const CRIME_SEVERITY_DETAILS: Record = {
+ [SERIOUS_CRIME_FILTER_NAME]: {
+ description: 'Violence, robbery, burglary and weapons possession near the postcode',
+ detail:
+ 'Combined count of the more serious street-level categories (violence and sexual offences, robbery, burglary, possession of weapons), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
+ },
+ [MINOR_CRIME_FILTER_NAME]: {
+ description: 'Anti-social behaviour, theft, criminal damage, drugs and public order near the postcode',
+ detail:
+ 'Combined count of the lower-severity street-level categories (anti-social behaviour, theft, criminal damage and arson, drugs, public order), as the average number of recorded incidents per year near the postcode. Toggle between the 7-year and the recent 2-year average.',
+ },
+};
+
+function keyPrefix(filterName: CrimeSeverityFilterName): string {
+ return `${filterName}:`;
+}
+
+/** The severity filter a name belongs to (from its key prefix or bare feature name), or null. */
+export function getCrimeSeverityFilterName(name: string): CrimeSeverityFilterName | null {
+ for (const filterName of CRIME_SEVERITY_FILTER_NAMES) {
+ if (name.startsWith(keyPrefix(filterName))) return filterName;
+ }
+ const type = getSpecificCrimeType(name);
+ return type && CRIME_SEVERITY_TYPE_SET.has(type) ? (type as CrimeSeverityFilterName) : null;
+}
+
+export function isCrimeSeverityFeatureName(name: string): boolean {
+ const type = getSpecificCrimeType(name);
+ return type != null && CRIME_SEVERITY_TYPE_SET.has(type);
+}
+
+export function isCrimeSeverityFilterName(name: string): boolean {
+ return getCrimeSeverityFilterName(name) != null;
+}
+
+export function createCrimeSeverityFilterKey(
+ filterName: CrimeSeverityFilterName,
+ featureName: string,
+ id: number | string
+): string {
+ return `${keyPrefix(filterName)}${encodeURIComponent(featureName)}:${id}`;
+}
+
+export function getCrimeSeverityFilterKeyId(name: string): string | null {
+ const filterName = getCrimeSeverityFilterName(name);
+ if (!filterName) return null;
+ const prefix = keyPrefix(filterName);
+ if (!name.startsWith(prefix)) return null; // a bare feature name has no id
+ const rest = name.substring(prefix.length);
+ const lastColon = rest.lastIndexOf(':');
+ return lastColon === -1 ? null : rest.substring(lastColon + 1);
+}
+
+export function parseCrimeSeverityFilterKey(name: string): string | null {
+ const filterName = getCrimeSeverityFilterName(name);
+ if (!filterName) return null;
+ const prefix = keyPrefix(filterName);
+ if (!name.startsWith(prefix)) return null;
+ const rest = name.substring(prefix.length);
+ const lastColon = rest.lastIndexOf(':');
+ if (lastColon === -1) return null;
+
+ const decoded = decodeURIComponent(rest.substring(0, lastColon));
+ return isCrimeSeverityFeatureName(decoded) ? decoded : null;
+}
+
+export function getCrimeSeverityFeatureName(name: string): string | null {
+ if (isCrimeSeverityFeatureName(name)) return name;
+ return parseCrimeSeverityFilterKey(name);
+}
+
+export function replaceCrimeSeverityFilterKeySelection(key: string, featureName: string): string {
+ const filterName = getCrimeSeverityFilterName(key) ?? getCrimeSeverityFilterName(featureName);
+ const id = getCrimeSeverityFilterKeyId(key) ?? '0';
+ // filterName is always resolvable here: the key carries a prefix and a window
+ // switch keeps the same severity type. Fall back defensively to the key as-is.
+ if (!filterName) return key;
+ return createCrimeSeverityFilterKey(filterName, featureName, id);
+}
+
+/** The default (7-year) backend feature for a severity, if present in `features`. */
+export function getDefaultCrimeSeverityFeatureName(
+ features: FeatureMeta[],
+ filterName: CrimeSeverityFilterName
+): string | null {
+ const name = specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT);
+ return features.some((feature) => feature.name === name) ? name : null;
+}
+
+export function normalizeCrimeSeverityFilters(filters: FeatureFilters): FeatureFilters {
+ let changed = false;
+ const next: FeatureFilters = {};
+
+ for (const [name, value] of Object.entries(filters)) {
+ if (isCrimeSeverityFeatureName(name)) {
+ const filterName = getCrimeSeverityFilterName(name)!;
+ next[createCrimeSeverityFilterKey(filterName, name, Object.keys(next).length)] = value;
+ changed = true;
+ continue;
+ }
+ next[name] = value;
+ }
+
+ return changed ? next : filters;
+}
+
+export function getCrimeSeverityFilterMeta(
+ features: FeatureMeta[],
+ filterName: CrimeSeverityFilterName
+): FeatureMeta {
+ const sourceFeatureName = getDefaultCrimeSeverityFeatureName(features, filterName);
+ const sourceFeature = sourceFeatureName
+ ? features.find((feature) => feature.name === sourceFeatureName)
+ : undefined;
+
+ return {
+ name: filterName,
+ type: 'numeric',
+ group: 'Crime',
+ min: sourceFeature?.min ?? 0,
+ max: sourceFeature?.max ?? 100,
+ step: 1,
+ description: CRIME_SEVERITY_DETAILS[filterName].description,
+ detail: CRIME_SEVERITY_DETAILS[filterName].detail,
+ source: 'crime',
+ suffix: '',
+ };
+}
+
+export function clampCrimeSeverityRange(
+ value: [number, number],
+ feature?: FeatureMeta
+): [number, number] {
+ return clampSpecificCrimeRange(value, feature);
+}
+
+/** The variant-filter config for one severity (single variant + 7y/2y window toggle). */
+export function getCrimeSeverityVariantConfig(
+ filterName: CrimeSeverityFilterName
+): VariantFilterConfig {
+ return {
+ filterName,
+ // One variant: the severity itself, in the default window. The card hides the
+ // (single-option) dropdown and exposes only the window toggle.
+ featureNames: [specificCrimeFeatureName(filterName, SPECIFIC_CRIME_WINDOW_DEFAULT)],
+ dropdownLabelKey: 'filters.crimeType',
+ getFilterMeta: (features) => getCrimeSeverityFilterMeta(features, filterName),
+ getDefaultFeatureName: (features) => getDefaultCrimeSeverityFeatureName(features, filterName),
+ getFeatureName: getCrimeSeverityFeatureName,
+ replaceFilterKeySelection: replaceCrimeSeverityFilterKeySelection,
+ clampRange: clampCrimeSeverityRange,
+ window: {
+ labelKey: 'filters.crimeWindow',
+ options: [
+ { id: '7y', labelKey: 'filters.crimeWindow7y' },
+ { id: '2y', labelKey: 'filters.crimeWindow2y' },
+ ],
+ getWindow: getSpecificCrimeWindow,
+ withWindow: (name, windowId) =>
+ withSpecificCrimeWindow(name, windowId as SpecificCrimeWindow),
+ },
+ };
+}
diff --git a/frontend/src/lib/crime-types.ts b/frontend/src/lib/crime-types.ts
index ff1f8d9..f6a082a 100644
--- a/frontend/src/lib/crime-types.ts
+++ b/frontend/src/lib/crime-types.ts
@@ -6,24 +6,47 @@
export interface CrimeTypeDef {
value: string;
+ /** English label (fallback / reference). */
label: string;
+ /** i18n key under `crimeTypes.*` used to render the selector label. */
+ labelKey: string;
}
export const CRIME_TYPES: readonly CrimeTypeDef[] = [
- { value: 'Violence and sexual offences', label: 'Violence & sexual offences' },
- { value: 'Anti-social behaviour', label: 'Anti-social behaviour' },
- { value: 'Criminal damage and arson', label: 'Criminal damage & arson' },
- { value: 'Public order', label: 'Public order' },
- { value: 'Shoplifting', label: 'Shoplifting' },
- { value: 'Vehicle crime', label: 'Vehicle crime' },
- { value: 'Burglary', label: 'Burglary' },
- { value: 'Other theft', label: 'Other theft' },
- { value: 'Theft from the person', label: 'Theft from the person' },
- { value: 'Bicycle theft', label: 'Bicycle theft' },
- { value: 'Drugs', label: 'Drugs' },
- { value: 'Robbery', label: 'Robbery' },
- { value: 'Possession of weapons', label: 'Possession of weapons' },
- { value: 'Other crime', label: 'Other crime' },
+ {
+ value: 'Violence and sexual offences',
+ label: 'Violence & sexual offences',
+ labelKey: 'crimeTypes.violenceAndSexualOffences',
+ },
+ {
+ value: 'Anti-social behaviour',
+ label: 'Anti-social behaviour',
+ labelKey: 'crimeTypes.antiSocialBehaviour',
+ },
+ {
+ value: 'Criminal damage and arson',
+ label: 'Criminal damage & arson',
+ labelKey: 'crimeTypes.criminalDamageAndArson',
+ },
+ { value: 'Public order', label: 'Public order', labelKey: 'crimeTypes.publicOrder' },
+ { value: 'Shoplifting', label: 'Shoplifting', labelKey: 'crimeTypes.shoplifting' },
+ { value: 'Vehicle crime', label: 'Vehicle crime', labelKey: 'crimeTypes.vehicleCrime' },
+ { value: 'Burglary', label: 'Burglary', labelKey: 'crimeTypes.burglary' },
+ { value: 'Other theft', label: 'Other theft', labelKey: 'crimeTypes.otherTheft' },
+ {
+ value: 'Theft from the person',
+ label: 'Theft from the person',
+ labelKey: 'crimeTypes.theftFromThePerson',
+ },
+ { value: 'Bicycle theft', label: 'Bicycle theft', labelKey: 'crimeTypes.bicycleTheft' },
+ { value: 'Drugs', label: 'Drugs', labelKey: 'crimeTypes.drugs' },
+ { value: 'Robbery', label: 'Robbery', labelKey: 'crimeTypes.robbery' },
+ {
+ value: 'Possession of weapons',
+ label: 'Possession of weapons',
+ labelKey: 'crimeTypes.possessionOfWeapons',
+ },
+ { value: 'Other crime', label: 'Other crime', labelKey: 'crimeTypes.otherCrime' },
] as const;
export const CRIME_TYPE_VALUES: readonly string[] = CRIME_TYPES.map((c) => c.value);
diff --git a/frontend/src/lib/feature-icons.tsx b/frontend/src/lib/feature-icons.tsx
index 215296e..a9da5f5 100644
--- a/frontend/src/lib/feature-icons.tsx
+++ b/frontend/src/lib/feature-icons.tsx
@@ -404,7 +404,12 @@ const FEATURE_ICON_PATHS: Record = {
* Returns a complete SVG icon element for a given feature name, or null if unmapped.
*/
export function getFeatureIcon(featureName: string, className: string): ReactElement | null {
- const paths = FEATURE_ICON_PATHS[featureName];
+ // Crime features ("Burglary (/yr, 7y)") share the bare type's legacy
+ // "(avg/yr)" icon key; both windows use the same icon.
+ const rateMatch = featureName.match(/^(.*) \(\/yr, \d+y\)$/);
+ const paths =
+ FEATURE_ICON_PATHS[featureName] ??
+ (rateMatch ? FEATURE_ICON_PATHS[`${rateMatch[1]} (avg/yr)`] : undefined);
if (!paths) return null;
return (
(QUALIFICATION_FEATURE_NAM
export function isQualificationFeatureName(name: string): boolean {
return QUALIFICATION_FEATURE_NAME_SET.has(name);
}
+
+export function isQualificationFilterName(name: string): boolean {
+ return isQualificationFeatureName(name) || name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX);
+}
+
+export function createQualificationFilterKey(featureName: string, id: number | string): string {
+ return `${QUALIFICATIONS_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
+}
+
+export function getQualificationFilterKeyId(name: string): string | null {
+ if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
+ const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
+ const lastColon = rest.lastIndexOf(':');
+ return lastColon === -1 ? null : rest.substring(lastColon + 1);
+}
+
+export function parseQualificationFilterKey(name: string): string | null {
+ if (!name.startsWith(QUALIFICATIONS_FILTER_KEY_PREFIX)) return null;
+ const rest = name.substring(QUALIFICATIONS_FILTER_KEY_PREFIX.length);
+ const lastColon = rest.lastIndexOf(':');
+ if (lastColon === -1) return null;
+
+ const decoded = decodeURIComponent(rest.substring(0, lastColon));
+ return isQualificationFeatureName(decoded) ? decoded : null;
+}
+
+export function getQualificationFeatureName(name: string): string | null {
+ if (isQualificationFeatureName(name)) return name;
+ return parseQualificationFilterKey(name);
+}
+
+export function replaceQualificationFilterKeySelection(key: string, featureName: string): string {
+ const id = getQualificationFilterKeyId(key) ?? '0';
+ return createQualificationFilterKey(featureName, id);
+}
+
+export function getDefaultQualificationFeatureName(features: FeatureMeta[]): string | null {
+ return (
+ QUALIFICATION_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ??
+ null
+ );
+}
+
+export function normalizeQualificationFilters(filters: FeatureFilters): FeatureFilters {
+ let changed = false;
+ const next: FeatureFilters = {};
+
+ for (const [name, value] of Object.entries(filters)) {
+ if (isQualificationFeatureName(name)) {
+ next[createQualificationFilterKey(name, Object.keys(next).length)] = value;
+ changed = true;
+ continue;
+ }
+ next[name] = value;
+ }
+
+ return changed ? next : filters;
+}
+
+export function getQualificationFilterMeta(features: FeatureMeta[]): FeatureMeta {
+ const sourceFeatureName = getDefaultQualificationFeatureName(features);
+ const sourceFeature = sourceFeatureName
+ ? features.find((feature) => feature.name === sourceFeatureName)
+ : undefined;
+
+ return {
+ name: QUALIFICATIONS_FILTER_NAME,
+ type: 'numeric',
+ group: 'Neighbours',
+ min: sourceFeature?.min ?? 0,
+ max: sourceFeature?.max ?? 100,
+ step: 1,
+ description:
+ 'Share of residents (16+) by highest qualification, from no qualifications to a degree or higher',
+ detail:
+ 'Filter by one Census 2021 (TS067) highest-qualification band at a time, e.g. the percentage of residents whose highest qualification is a degree or higher.',
+ source: 'census-2021',
+ suffix: '%',
+ };
+}
+
+export function clampQualificationRange(
+ value: [number, number],
+ feature?: FeatureMeta
+): [number, number] {
+ const min = feature?.histogram?.min ?? feature?.min ?? 0;
+ const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
+ return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
+}
+
+export const QUALIFICATION_VARIANT_CONFIG: VariantFilterConfig = {
+ filterName: QUALIFICATIONS_FILTER_NAME,
+ featureNames: QUALIFICATION_FEATURE_NAMES,
+ dropdownLabelKey: 'filters.qualificationLevel',
+ getFilterMeta: getQualificationFilterMeta,
+ getDefaultFeatureName: getDefaultQualificationFeatureName,
+ getFeatureName: getQualificationFeatureName,
+ replaceFilterKeySelection: replaceQualificationFilterKeySelection,
+ clampRange: clampQualificationRange,
+};
diff --git a/frontend/src/lib/tenure-filter.ts b/frontend/src/lib/tenure-filter.ts
new file mode 100644
index 0000000..6527f15
--- /dev/null
+++ b/frontend/src/lib/tenure-filter.ts
@@ -0,0 +1,121 @@
+import type { FeatureFilters, FeatureMeta } from '../types';
+import type { VariantFilterConfig } from './variant-filter';
+
+export const TENURE_FILTER_NAME = 'Tenure';
+export const TENURE_FILTER_KEY_PREFIX = `${TENURE_FILTER_NAME}:`;
+
+/**
+ * The Census 2021 housing tenure categories (TS054). They sum to 100% per
+ * neighbourhood and render as a single stacked "Tenure" composition in the area
+ * pane (see STACKED_GROUPS["Neighbours"] in consts.ts). In the filter browser
+ * they fold into one "Tenure" filter whose dropdown selects a category —
+ * owner-occupied, social rent or private rent — rather than three separate
+ * sliders.
+ */
+export const TENURE_FEATURE_NAMES = [
+ '% Owner occupied',
+ '% Social rent',
+ '% Private rent',
+] as const;
+
+const TENURE_FEATURE_NAME_SET = new Set(TENURE_FEATURE_NAMES);
+
+export function isTenureFeatureName(name: string): boolean {
+ return TENURE_FEATURE_NAME_SET.has(name);
+}
+
+export function isTenureFilterName(name: string): boolean {
+ return isTenureFeatureName(name) || name.startsWith(TENURE_FILTER_KEY_PREFIX);
+}
+
+export function createTenureFilterKey(featureName: string, id: number | string): string {
+ return `${TENURE_FILTER_KEY_PREFIX}${encodeURIComponent(featureName)}:${id}`;
+}
+
+export function getTenureFilterKeyId(name: string): string | null {
+ if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
+ const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
+ const lastColon = rest.lastIndexOf(':');
+ return lastColon === -1 ? null : rest.substring(lastColon + 1);
+}
+
+export function parseTenureFilterKey(name: string): string | null {
+ if (!name.startsWith(TENURE_FILTER_KEY_PREFIX)) return null;
+ const rest = name.substring(TENURE_FILTER_KEY_PREFIX.length);
+ const lastColon = rest.lastIndexOf(':');
+ if (lastColon === -1) return null;
+
+ const decoded = decodeURIComponent(rest.substring(0, lastColon));
+ return isTenureFeatureName(decoded) ? decoded : null;
+}
+
+export function getTenureFeatureName(name: string): string | null {
+ if (isTenureFeatureName(name)) return name;
+ return parseTenureFilterKey(name);
+}
+
+export function replaceTenureFilterKeySelection(key: string, featureName: string): string {
+ const id = getTenureFilterKeyId(key) ?? '0';
+ return createTenureFilterKey(featureName, id);
+}
+
+export function getDefaultTenureFeatureName(features: FeatureMeta[]): string | null {
+ return (
+ TENURE_FEATURE_NAMES.find((name) => features.some((feature) => feature.name === name)) ?? null
+ );
+}
+
+export function normalizeTenureFilters(filters: FeatureFilters): FeatureFilters {
+ let changed = false;
+ const next: FeatureFilters = {};
+
+ for (const [name, value] of Object.entries(filters)) {
+ if (isTenureFeatureName(name)) {
+ next[createTenureFilterKey(name, Object.keys(next).length)] = value;
+ changed = true;
+ continue;
+ }
+ next[name] = value;
+ }
+
+ return changed ? next : filters;
+}
+
+export function getTenureFilterMeta(features: FeatureMeta[]): FeatureMeta {
+ const sourceFeatureName = getDefaultTenureFeatureName(features);
+ const sourceFeature = sourceFeatureName
+ ? features.find((feature) => feature.name === sourceFeatureName)
+ : undefined;
+
+ return {
+ name: TENURE_FILTER_NAME,
+ type: 'numeric',
+ group: 'Neighbours',
+ min: sourceFeature?.min ?? 0,
+ max: sourceFeature?.max ?? 100,
+ step: 1,
+ description:
+ 'Share of households that own their home, rent from a social landlord, or rent privately',
+ detail:
+ 'Filter by one Census 2021 (TS054) housing tenure category at a time, e.g. the percentage of households that own their home.',
+ source: 'census-2021',
+ suffix: '%',
+ };
+}
+
+export function clampTenureRange(value: [number, number], feature?: FeatureMeta): [number, number] {
+ const min = feature?.histogram?.min ?? feature?.min ?? 0;
+ const max = feature?.histogram?.max ?? feature?.max ?? Math.max(1, value[1]);
+ return [Math.max(min, Math.min(value[0], max)), Math.max(min, Math.min(value[1], max))];
+}
+
+export const TENURE_VARIANT_CONFIG: VariantFilterConfig = {
+ filterName: TENURE_FILTER_NAME,
+ featureNames: TENURE_FEATURE_NAMES,
+ dropdownLabelKey: 'filters.tenureType',
+ getFilterMeta: getTenureFilterMeta,
+ getDefaultFeatureName: getDefaultTenureFeatureName,
+ getFeatureName: getTenureFeatureName,
+ replaceFilterKeySelection: replaceTenureFilterKeySelection,
+ clampRange: clampTenureRange,
+};
diff --git a/frontend/src/lib/url-state.test.ts b/frontend/src/lib/url-state.test.ts
index 483e2b5..ab389fe 100644
--- a/frontend/src/lib/url-state.test.ts
+++ b/frontend/src/lib/url-state.test.ts
@@ -5,8 +5,11 @@ import { parseUrlState, stateToParams } from './url-state';
import { INITIAL_VIEW_STATE } from './consts';
import { createSchoolFilterKey } from './school-filter';
import { createSpecificCrimeFilterKey } from './crime-filter';
+import { createCrimeSeverityFilterKey } from './crime-severity-filter';
import { createElectionVoteShareFilterKey } from './election-filter';
import { createEthnicityFilterKey } from './ethnicity-filter';
+import { createQualificationFilterKey } from './qualification-filter';
+import { createTenureFilterKey } from './tenure-filter';
import {
POI_COUNT_2KM_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME,
@@ -405,8 +408,8 @@ describe('url-state', () => {
});
it('round-trips repeated specific crime filters with dedicated URL params', () => {
- const burglary = createSpecificCrimeFilterKey('Burglary (avg/yr)', 1);
- const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 2);
+ const burglary = createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 1);
+ const vehicleCrime = createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 2);
const params = stateToParams(
null,
@@ -420,8 +423,8 @@ describe('url-state', () => {
);
expect(params.getAll('crime')).toEqual([
- 'Burglary (avg/yr):0:5',
- 'Vehicle crime (avg/yr):1:10',
+ 'Burglary (/yr, 7y):0:5',
+ 'Vehicle crime (/yr, 7y):1:10',
]);
expect(params.getAll('filter')).toEqual([]);
@@ -429,8 +432,42 @@ describe('url-state', () => {
const state = parseUrlState();
expect(state.filters).toEqual({
- [createSpecificCrimeFilterKey('Burglary (avg/yr)', 0)]: [0, 5],
- [createSpecificCrimeFilterKey('Vehicle crime (avg/yr)', 1)]: [1, 10],
+ [createSpecificCrimeFilterKey('Burglary (/yr, 7y)', 0)]: [0, 5],
+ [createSpecificCrimeFilterKey('Vehicle crime (/yr, 7y)', 1)]: [1, 10],
+ });
+ });
+
+ it('round-trips serious/minor crime severity filters with dedicated URL params', () => {
+ const serious = createCrimeSeverityFilterKey(
+ 'Serious crime',
+ 'Serious crime (/yr, 7y)',
+ 0
+ );
+ const minor = createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1);
+
+ const params = stateToParams(
+ null,
+ {
+ [serious]: [0, 12],
+ [minor]: [3, 40],
+ },
+ [],
+ new Set(),
+ 'area'
+ );
+
+ expect(params.getAll('crimeSeverity')).toEqual([
+ 'Serious crime (/yr, 7y):0:12',
+ 'Minor crime (/yr, 2y):3:40',
+ ]);
+ expect(params.getAll('filter')).toEqual([]);
+
+ window.history.replaceState({}, '', `/?${params.toString()}`);
+ const state = parseUrlState();
+
+ expect(state.filters).toEqual({
+ [createCrimeSeverityFilterKey('Serious crime', 'Serious crime (/yr, 7y)', 0)]: [0, 12],
+ [createCrimeSeverityFilterKey('Minor crime', 'Minor crime (/yr, 2y)', 1)]: [3, 40],
});
});
@@ -488,6 +525,65 @@ describe('url-state', () => {
});
});
+ it('round-trips repeated qualification filters with dedicated URL params', () => {
+ // "% Degree or higher" exercises the leading-`%` + space encoding path.
+ const degree = createQualificationFilterKey('% Degree or higher', 1);
+ const noQuals = createQualificationFilterKey('% No qualifications', 2);
+
+ const params = stateToParams(
+ null,
+ {
+ [degree]: [20, 60],
+ [noQuals]: [0, 25],
+ },
+ [],
+ new Set(),
+ 'area'
+ );
+
+ expect(params.getAll('qualification')).toEqual([
+ '% Degree or higher:20:60',
+ '% No qualifications:0:25',
+ ]);
+ expect(params.getAll('filter')).toEqual([]);
+
+ window.history.replaceState({}, '', `/?${params.toString()}`);
+ const state = parseUrlState();
+
+ expect(state.filters).toEqual({
+ [createQualificationFilterKey('% Degree or higher', 0)]: [20, 60],
+ [createQualificationFilterKey('% No qualifications', 1)]: [0, 25],
+ });
+ });
+
+ it('round-trips repeated tenure filters with dedicated URL params', () => {
+ // "% Owner occupied" exercises the leading-`%` + space encoding path.
+ const owner = createTenureFilterKey('% Owner occupied', 1);
+ const privateRent = createTenureFilterKey('% Private rent', 2);
+
+ const params = stateToParams(
+ null,
+ {
+ [owner]: [20, 60],
+ [privateRent]: [0, 25],
+ },
+ [],
+ new Set(),
+ 'area'
+ );
+
+ expect(params.getAll('tenure')).toEqual(['% Owner occupied:20:60', '% Private rent:0:25']);
+ expect(params.getAll('filter')).toEqual([]);
+
+ window.history.replaceState({}, '', `/?${params.toString()}`);
+ const state = parseUrlState();
+
+ expect(state.filters).toEqual({
+ [createTenureFilterKey('% Owner occupied', 0)]: [20, 60],
+ [createTenureFilterKey('% Private rent', 1)]: [0, 25],
+ });
+ });
+
it('round-trips repeated amenity distance filters with dedicated URL params', () => {
const park = createPoiDistanceFilterKey('Distance to nearest amenity (Park) (km)', 3);
const cafe = createPoiDistanceFilterKey('Distance to nearest amenity (Café) (km)', 4);
diff --git a/frontend/src/lib/url-state.ts b/frontend/src/lib/url-state.ts
index 20bfe22..ae3fd89 100644
--- a/frontend/src/lib/url-state.ts
+++ b/frontend/src/lib/url-state.ts
@@ -22,6 +22,13 @@ import {
isSpecificCrimeFeatureName,
isSpecificCrimeFilterName,
} from './crime-filter';
+import {
+ createCrimeSeverityFilterKey,
+ getCrimeSeverityFeatureName,
+ getCrimeSeverityFilterName,
+ isCrimeSeverityFeatureName,
+ isCrimeSeverityFilterName,
+} from './crime-severity-filter';
import {
ELECTION_VOTE_SHARE_FILTER_NAME,
createElectionVoteShareFilterKey,
@@ -36,6 +43,20 @@ import {
isEthnicityFeatureName,
isEthnicityFilterName,
} from './ethnicity-filter';
+import {
+ QUALIFICATIONS_FILTER_NAME,
+ createQualificationFilterKey,
+ getQualificationFeatureName,
+ isQualificationFeatureName,
+ isQualificationFilterName,
+} from './qualification-filter';
+import {
+ TENURE_FILTER_NAME,
+ createTenureFilterKey,
+ getTenureFeatureName,
+ isTenureFeatureName,
+ isTenureFilterName,
+} from './tenure-filter';
import {
POI_DISTANCE_FILTER_NAME,
TRANSPORT_DISTANCE_FILTER_NAME,
@@ -83,8 +104,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
const filterParams = params.getAll('filter');
const schoolParams = params.getAll('school');
const crimeParams = params.getAll('crime');
+ const crimeSeverityParams = params.getAll('crimeSeverity');
const voteShareParams = params.getAll('voteShare');
const ethnicityParams = params.getAll('ethnicity');
+ const qualificationParams = params.getAll('qualification');
+ const tenureParams = params.getAll('tenure');
const amenityDistanceParams = params.getAll('amenityDistance');
const transportDistanceParams = params.getAll('transportDistance');
const amenityCount2KmParams = params.getAll('amenityCount2km');
@@ -93,8 +117,11 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filterParams.length === 0 &&
schoolParams.length === 0 &&
crimeParams.length === 0 &&
+ crimeSeverityParams.length === 0 &&
voteShareParams.length === 0 &&
ethnicityParams.length === 0 &&
+ qualificationParams.length === 0 &&
+ tenureParams.length === 0 &&
amenityDistanceParams.length === 0 &&
transportDistanceParams.length === 0 &&
amenityCount2KmParams.length === 0 &&
@@ -155,6 +182,19 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filters[createSpecificCrimeFilterKey(featureName, index)] = [min, max];
});
+ crimeSeverityParams.forEach((entry, index) => {
+ const parts = entry.split(':');
+ if (parts.length < 3) return;
+ const featureName = parts.slice(0, -2).join(':');
+ const min = Number(parts[parts.length - 2]);
+ const max = Number(parts[parts.length - 1]);
+ const filterName = getCrimeSeverityFilterName(featureName);
+ if (!isCrimeSeverityFeatureName(featureName) || !filterName || isNaN(min) || isNaN(max)) {
+ return;
+ }
+ filters[createCrimeSeverityFilterKey(filterName, featureName, index)] = [min, max];
+ });
+
voteShareParams.forEach((entry, index) => {
const parts = entry.split(':');
if (parts.length < 3) return;
@@ -179,6 +219,30 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
filters[createEthnicityFilterKey(featureName, index)] = [min, max];
});
+ qualificationParams.forEach((entry, index) => {
+ const parts = entry.split(':');
+ if (parts.length < 3) return;
+ const featureName = parts.slice(0, -2).join(':');
+ const min = Number(parts[parts.length - 2]);
+ const max = Number(parts[parts.length - 1]);
+ if (!isQualificationFeatureName(featureName) || isNaN(min) || isNaN(max)) {
+ return;
+ }
+ filters[createQualificationFilterKey(featureName, index)] = [min, max];
+ });
+
+ tenureParams.forEach((entry, index) => {
+ const parts = entry.split(':');
+ if (parts.length < 3) return;
+ const featureName = parts.slice(0, -2).join(':');
+ const min = Number(parts[parts.length - 2]);
+ const max = Number(parts[parts.length - 1]);
+ if (!isTenureFeatureName(featureName) || isNaN(min) || isNaN(max)) {
+ return;
+ }
+ filters[createTenureFilterKey(featureName, index)] = [min, max];
+ });
+
const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => {
entries.forEach((entry, index) => {
const parts = entry.split(':');
@@ -401,6 +465,13 @@ export function stateToParams(
continue;
}
+ const crimeSeverityFeatureName = getCrimeSeverityFeatureName(name);
+ if (crimeSeverityFeatureName && isCrimeSeverityFilterName(name)) {
+ const [min, max] = value as [number, number];
+ params.append('crimeSeverity', `${crimeSeverityFeatureName}:${min}:${max}`);
+ continue;
+ }
+
const electionVoteShareFeatureName = getElectionVoteShareFeatureName(name);
if (electionVoteShareFeatureName && isElectionVoteShareFilterName(name)) {
const [min, max] = value as [number, number];
@@ -415,6 +486,20 @@ export function stateToParams(
continue;
}
+ const qualificationFeatureName = getQualificationFeatureName(name);
+ if (qualificationFeatureName && isQualificationFilterName(name)) {
+ const [min, max] = value as [number, number];
+ params.append('qualification', `${qualificationFeatureName}:${min}:${max}`);
+ continue;
+ }
+
+ const tenureFeatureName = getTenureFeatureName(name);
+ if (tenureFeatureName && isTenureFilterName(name)) {
+ const [min, max] = value as [number, number];
+ params.append('tenure', `${tenureFeatureName}:${min}:${max}`);
+ continue;
+ }
+
const amenityDistanceFeatureName = getPoiDistanceFeatureName(name);
if (amenityDistanceFeatureName && isPoiDistanceFilterName(name)) {
const [min, max] = value as [number, number];
@@ -522,8 +607,11 @@ export function summarizeParams(queryString: string): string {
const filterParams = params.getAll('filter');
const schoolParams = params.getAll('school');
const crimeParams = params.getAll('crime');
+ const crimeSeverityParams = params.getAll('crimeSeverity');
const voteShareParams = params.getAll('voteShare');
const ethnicityParams = params.getAll('ethnicity');
+ const qualificationParams = params.getAll('qualification');
+ const tenureParams = params.getAll('tenure');
const amenityDistanceParams = params.getAll('amenityDistance');
const transportDistanceParams = params.getAll('transportDistance');
const amenityCount2KmParams = params.getAll('amenityCount2km');
@@ -532,8 +620,11 @@ export function summarizeParams(queryString: string): string {
filterParams.length > 0 ||
schoolParams.length > 0 ||
crimeParams.length > 0 ||
+ crimeSeverityParams.length > 0 ||
voteShareParams.length > 0 ||
ethnicityParams.length > 0 ||
+ qualificationParams.length > 0 ||
+ tenureParams.length > 0 ||
amenityDistanceParams.length > 0 ||
transportDistanceParams.length > 0 ||
amenityCount2KmParams.length > 0 ||
@@ -544,8 +635,11 @@ export function summarizeParams(queryString: string): string {
const colonIdx = entry.indexOf(':');
const name = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
+ if (isCrimeSeverityFeatureName(name)) return getCrimeSeverityFilterName(name) ?? name;
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
+ if (isQualificationFeatureName(name)) return QUALIFICATIONS_FILTER_NAME;
+ if (isTenureFeatureName(name)) return TENURE_FILTER_NAME;
const poiFilterName = getPoiFilterName(name);
if (poiFilterName) return poiFilterName;
return name;
@@ -555,12 +649,24 @@ export function summarizeParams(queryString: string): string {
for (let i = 0; i < crimeParams.length; i++) {
filterNames.push(SPECIFIC_CRIMES_FILTER_NAME);
}
+ for (const entry of crimeSeverityParams) {
+ const colonIdx = entry.indexOf(':');
+ const featureName = colonIdx > 0 ? entry.substring(0, colonIdx) : entry;
+ const severityFilterName = getCrimeSeverityFilterName(featureName);
+ if (severityFilterName) filterNames.push(severityFilterName);
+ }
for (let i = 0; i < voteShareParams.length; i++) {
filterNames.push(ELECTION_VOTE_SHARE_FILTER_NAME);
}
for (let i = 0; i < ethnicityParams.length; i++) {
filterNames.push(ETHNICITIES_FILTER_NAME);
}
+ for (let i = 0; i < qualificationParams.length; i++) {
+ filterNames.push(QUALIFICATIONS_FILTER_NAME);
+ }
+ for (let i = 0; i < tenureParams.length; i++) {
+ filterNames.push(TENURE_FILTER_NAME);
+ }
for (let i = 0; i < amenityDistanceParams.length; i++) {
filterNames.push(POI_DISTANCE_FILTER_NAME);
}
diff --git a/frontend/src/lib/variant-filter.ts b/frontend/src/lib/variant-filter.ts
new file mode 100644
index 0000000..b49294d
--- /dev/null
+++ b/frontend/src/lib/variant-filter.ts
@@ -0,0 +1,80 @@
+import type { FeatureMeta } from '../types';
+
+/** i18n keys (typed for the strict `t()`) usable as a variant dropdown label. */
+type VariantDropdownLabelKey =
+ | 'filters.crimeType'
+ | 'filters.qualificationLevel'
+ | 'filters.tenureType';
+
+/** i18n keys (typed for the strict `t()`) usable as a window-toggle label. */
+type VariantWindowLabelKey =
+ | 'filters.crimeWindow'
+ | 'filters.crimeWindow7y'
+ | 'filters.crimeWindow2y';
+
+/** One option in a variant filter's secondary window/period toggle. */
+export interface VariantWindowOption {
+ /** Stable id encoded inside the feature name (e.g. "7y"). */
+ id: string;
+ /** i18n key for the toggle button label. */
+ labelKey: VariantWindowLabelKey;
+}
+
+/**
+ * Optional secondary axis for a variant filter: the same variant measured over
+ * a different time window (e.g. crime rates over 7 vs 2 years). The dropdown
+ * still picks the variant; this toggle picks the window. Both ultimately select
+ * a single backend feature name, so switching either re-points the filter key.
+ */
+export interface VariantWindowConfig {
+ /** Toggle options in display order. */
+ options: VariantWindowOption[];
+ /** Optional i18n key for a small label above the toggle. */
+ labelKey?: VariantWindowLabelKey;
+ /** Window id of a feature name (e.g. "7y"), or null if unrecognized. */
+ getWindow: (featureName: string) => string | null;
+ /** The same variant's feature name in a different window. */
+ withWindow: (featureName: string, windowId: string) => string;
+}
+
+/**
+ * Shared shape for the "pick one backend feature variant, filter its range"
+ * family of client-side aggregate filters. Several Census/police breakdowns
+ * (specific crimes, qualifications, …) are dozens of individual percentage
+ * features that would clutter the filter browser as separate sliders. Instead
+ * each is folded into a single named filter whose card carries a dropdown to
+ * choose the variant, reusing one component ([`VariantFilterCard`]).
+ *
+ * Each filter library (e.g. `crime-filter`, `qualification-filter`) exports a
+ * config so the card stays variant-agnostic.
+ */
+export interface VariantFilterConfig {
+ /** Display name of the aggregate filter, e.g. "Specific crimes". */
+ filterName: string;
+ /**
+ * Backend feature names enumerating the dropdown options, in display order.
+ * With a [`window`] config these are the canonical (default-window) names;
+ * the card re-points each to the currently selected window.
+ */
+ featureNames: readonly string[];
+ /** i18n key for the dropdown label, e.g. "filters.crimeType". */
+ dropdownLabelKey: VariantDropdownLabelKey;
+ /** Synthetic [`FeatureMeta`] used for the aggregate filter's own label/info. */
+ getFilterMeta: (features: FeatureMeta[]) => FeatureMeta;
+ /** First selectable variant present in `features`, or null if none. */
+ getDefaultFeatureName: (features: FeatureMeta[]) => string | null;
+ /** Backend feature name for a filter key (or a bare feature name). */
+ getFeatureName: (name: string) => string | null;
+ /** Rewrite a filter key to point at a different variant, keeping its id. */
+ replaceFilterKeySelection: (key: string, featureName: string) => string;
+ /** Clamp a [min, max] range into the selected feature's bounds. */
+ clampRange: (value: [number, number], feature?: FeatureMeta) => [number, number];
+ /**
+ * Server-translatable source value for a dropdown option's label; the card
+ * renders `ts(...)` of it. Defaults to the feature name itself. Useful with a
+ * [`window`] toggle so the label can be the bare variant (no window suffix).
+ */
+ getOptionLabelSource?: (featureName: string) => string;
+ /** Optional secondary window/period toggle (e.g. 7- vs 2-year crime rates). */
+ window?: VariantWindowConfig;
+}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index bf648ed..0db2755 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -316,26 +316,17 @@ export interface CrimeYearStats {
}
export interface CrimeAreaAverage {
- /** Full rate-feature name (e.g. "Burglary (per 1k/yr, 7y)"). */
+ /** Full crime-feature name (e.g. "Burglary (/yr, 7y)"). */
name: string;
- /** Exact national mean rate. Preferred over the histogram-bin national
+ /** Exact national mean count. Preferred over the histogram-bin national
* average for crime so all reference numbers share one estimator. */
national?: number;
- /** Mean rate across the selection's outcode. */
+ /** Mean count across the selection's outcode. */
outcode?: number;
- /** Mean rate across the selection's postcode sector. */
+ /** Mean count across the selection's postcode sector. */
sector?: number;
}
-export interface CrimeRawStats {
- /** Bare crime type (e.g. "Burglary"). */
- name: string;
- /** Mean recorded incidents/yr over the last 7 years. */
- per_yr_7y?: number;
- /** Mean recorded incidents/yr over the last 2 years. */
- per_yr_2y?: number;
-}
-
/** One individual police.uk crime, from /api/crime-records. */
export interface CrimeIncident {
/** "YYYY-MM". */
@@ -352,6 +343,11 @@ export interface CrimeRecordsResponse {
records: CrimeIncident[];
total: number;
offset: number;
+ /**
+ * Server flag meaning "more pages exist beyond this one". The client derives
+ * pagination from `total` vs `records.length`, so this is currently not
+ * surfaced in the UI (kept to mirror the server response shape).
+ */
truncated: boolean;
}
@@ -386,12 +382,9 @@ export interface HexagonStatsResponse {
/** Postcode sector (e.g. "E14 2") of the selection's central postcode, when
* sector crime averages are available for it. */
crime_sector?: string;
- /** Per-rate-feature average rates across the central postcode's outcode and
+ /** Per-crime-feature average counts across the central postcode's outcode and
* sector, shown alongside the national average for each crime metric. */
crime_area_averages?: CrimeAreaAverage[];
- /** Raw (un-normalised) recorded incidents/yr per crime type, shown beside the
- * normalised rate. Display-only. */
- crime_raw?: CrimeRawStats[];
/** Total individual crime records (last 7 years) across the selection's
* postcodes — the count behind the "individual crimes" list. */
crime_total_records?: number;
diff --git a/pipeline/transform/area_crime_averages.py b/pipeline/transform/area_crime_averages.py
new file mode 100644
index 0000000..2b86b0d
--- /dev/null
+++ b/pipeline/transform/area_crime_averages.py
@@ -0,0 +1,212 @@
+"""Precompute per-outcode / per-postcode-sector / national mean headline crime
+counts for the right pane's area comparison.
+
+The right pane shows each crime metric next to its area context: the mean
+average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
+``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
+within a postcode (the merge keys it on the postcode), so each postcode
+contributes its single value weighted by how many properties sit in it — keeping
+every scope on the same property-weighted basis as the per-selection mean, so the
+four numbers (this selection / sector / outcode / nation) are directly
+comparable. The national figure here is an EXACT property-weighted mean, which is
+why it overrides the upward-biased histogram-bin national average for crime.
+
+This used to be recomputed inside the server on every boot from the loaded
+property matrix. It is a pure function of the two merge outputs, so it belongs in
+the data build; the server now just loads the parquet this writes. Reading the
+crime values from ``postcode.parquet`` and the per-postcode property weights from
+``properties.parquet`` mirrors exactly the two inputs the server loads, so the
+result matches what the server used to compute (minus its u16 quantization loss).
+
+Output schema — one row per area:
+
+ scope : ``"national"`` | ``"outcode"`` | ``"sector"``
+ area : the outcode (``"E14"``) / sector (``"E14 2"``);
+ ``""`` for the single national row
+ `` (/yr, 7y|2y)`` : Float32 property-weighted mean crime count per year
+ (null = the scope has no data for that type)
+"""
+
+import argparse
+from pathlib import Path
+
+import polars as pl
+
+# Filterable crime columns are the average-annual incident counts and carry this
+# marker in postcode.parquet (e.g. "Burglary (/yr, 7y)"). We average those. The
+# full column name is kept; the server discovers and keys area averages by the
+# same names.
+COUNT_MARKER = " (/yr, "
+
+# `scope` discriminator values. The server's loader dispatches on these.
+SCOPE_NATIONAL = "national"
+SCOPE_OUTCODE = "outcode"
+SCOPE_SECTOR = "sector"
+
+# Area label on the national row — it spans the whole country, so it has no code.
+NATIONAL_AREA = ""
+
+# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
+# "E14 2DG").
+POSTCODE_COLUMN = "Postcode"
+
+# Internal weight / split columns dropped before write.
+_WEIGHT_COLUMN = "_weight"
+_OUTCODE_COLUMN = "_outcode"
+_SECTOR_COLUMN = "_sector"
+
+
+def _crime_columns(columns: list[str]) -> list[str]:
+ crime_cols = [name for name in columns if COUNT_MARKER in name]
+ if not crime_cols:
+ raise ValueError(
+ f"postcode parquet has no '*{COUNT_MARKER}*' crime count columns to average"
+ )
+ return crime_cols
+
+
+def _weighted_mean(column: str) -> pl.Expr:
+ """Property-weighted mean of ``column`` that excludes nulls from BOTH the
+ value sum and the weight.
+
+ A null crime value is a genuine gap (the postcode's police force published no
+ usable data), not zero crime, so it must dilute neither the numerator nor the
+ denominator — exactly as the server's former estimator skipped NaN values.
+ Yields null when no postcode in the group has data for this type.
+ """
+ weight = pl.col(_WEIGHT_COLUMN)
+ numerator = (pl.col(column) * weight).sum()
+ denominator = weight.filter(pl.col(column).is_not_null()).sum()
+ return (
+ pl.when(denominator > 0)
+ .then(numerator / denominator)
+ .otherwise(None)
+ .cast(pl.Float32)
+ .alias(column)
+ )
+
+
+def compute_area_crime_averages(
+ postcodes: pl.LazyFrame, properties: pl.LazyFrame
+) -> pl.DataFrame:
+ """Build the national / per-outcode / per-sector crime-average table.
+
+ ``postcodes`` is the merge's postcode output (one row per active postcode,
+ carrying the ``"* (/yr, *)"`` crime count columns); ``properties`` is the merge's
+ per-property output, used only to weight each postcode by its property count.
+ """
+ crime_cols = _crime_columns(postcodes.collect_schema().names())
+
+ # Property weight per postcode = how many property rows the server indexes
+ # under it. The inner join keeps only postcodes that actually carry
+ # properties, matching the server's per-postcode row index (a postcode with
+ # no properties never contributed to any average).
+ weights = properties.group_by(POSTCODE_COLUMN).agg(pl.len().alias(_WEIGHT_COLUMN))
+
+ # Outcode / sector of the spaced `pcds` postcode, matching the server's
+ # postcode_outcode / postcode_sector (split on the single space; sector =
+ # outcode + space + first inward character). Null where the form has no
+ # inward code, so such rows drop out of the per-area groups.
+ parts = pl.col(POSTCODE_COLUMN).str.splitn(" ", 2).struct
+ outward = parts.field("field_0")
+ inward = parts.field("field_1")
+ base = (
+ postcodes.select(POSTCODE_COLUMN, *crime_cols)
+ .join(weights, on=POSTCODE_COLUMN, how="inner")
+ .with_columns(
+ pl.when(inward.is_not_null())
+ .then(outward)
+ .otherwise(None)
+ .alias(_OUTCODE_COLUMN),
+ pl.when(inward.str.len_chars() >= 1)
+ .then(outward + pl.lit(" ") + inward.str.slice(0, 1))
+ .otherwise(None)
+ .alias(_SECTOR_COLUMN),
+ )
+ )
+
+ mean_exprs = [_weighted_mean(column) for column in crime_cols]
+
+ national = base.select(
+ pl.lit(SCOPE_NATIONAL).alias("scope"),
+ pl.lit(NATIONAL_AREA).alias("area"),
+ *mean_exprs,
+ )
+ by_outcode = (
+ base.drop_nulls(_OUTCODE_COLUMN)
+ .group_by(_OUTCODE_COLUMN)
+ .agg(mean_exprs)
+ .select(
+ pl.lit(SCOPE_OUTCODE).alias("scope"),
+ pl.col(_OUTCODE_COLUMN).alias("area"),
+ *crime_cols,
+ )
+ )
+ by_sector = (
+ base.drop_nulls(_SECTOR_COLUMN)
+ .group_by(_SECTOR_COLUMN)
+ .agg(mean_exprs)
+ .select(
+ pl.lit(SCOPE_SECTOR).alias("scope"),
+ pl.col(_SECTOR_COLUMN).alias("area"),
+ *crime_cols,
+ )
+ )
+
+ result = pl.concat([national, by_outcode, by_sector], how="vertical").collect(
+ engine="streaming"
+ )
+
+ # Drop per-area rows where every crime type is null: the server only created
+ # a map entry once a scope had at least one finite value, so an all-null
+ # outcode/sector reported no code at all. The national row is always kept (it
+ # always has data, and is emitted even for areas absent from both maps).
+ has_any = pl.any_horizontal(pl.col(column).is_not_null() for column in crime_cols)
+ return result.filter((pl.col("scope") == SCOPE_NATIONAL) | has_any)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description=(
+ "Precompute national / per-outcode / per-sector mean headline crime "
+ "counts from the merge outputs"
+ )
+ )
+ parser.add_argument(
+ "--postcodes",
+ type=Path,
+ required=True,
+ help="postcode.parquet (area features, incl. the '* (/yr, *)' crime columns)",
+ )
+ parser.add_argument(
+ "--properties",
+ type=Path,
+ required=True,
+ help="properties.parquet (per-property rows; supplies postcode property weights)",
+ )
+ parser.add_argument(
+ "--output",
+ type=Path,
+ required=True,
+ help="Output area_crime_averages.parquet path",
+ )
+ args = parser.parse_args()
+
+ result = compute_area_crime_averages(
+ pl.scan_parquet(args.postcodes), pl.scan_parquet(args.properties)
+ )
+ outcodes = result.filter(pl.col("scope") == SCOPE_OUTCODE).height
+ sectors = result.filter(pl.col("scope") == SCOPE_SECTOR).height
+ print(
+ f"Area crime averages: {result.height} rows "
+ f"({outcodes} outcodes, {sectors} sectors, "
+ f"{len(_crime_columns(result.columns))} crime types)"
+ )
+
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ result.write_parquet(args.output, compression="zstd")
+ print(f"Saved to {args.output}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pipeline/transform/crime_spatial.py b/pipeline/transform/crime_spatial.py
index 9a061ef..e500b1d 100644
--- a/pipeline/transform/crime_spatial.py
+++ b/pipeline/transform/crime_spatial.py
@@ -2,22 +2,26 @@
Instead of attributing each incident to its published LSOA code, this transform
counts the anonymised incident *points* that fall within ``buffer_m`` (default
-100m) of each postcode's boundary polygon (the polygon buffered outward). A point
+50m) of each postcode's boundary polygon (the polygon buffered outward). A point
inside several overlapping buffers counts for each postcode -- the same
multiplicity the tree-density filter uses for features near more than one
-postcode. The wide 100m buffer deliberately smooths police.uk's snap-to-grid
+postcode. The 50m buffer deliberately smooths police.uk's snap-to-grid
coordinates, which would otherwise make the count hypersensitive to which side of
a narrow line a shared "map point" anchor happened to land on.
-Counts are **area-normalised**: each postcode's count is divided by its buffered
-catchment area and rescaled by the median catchment area, so the metric reflects
-crime *density* rather than how much ground the buffer sweeps (a median-sized
-catchment is left unchanged; a large rural postcode is no longer inflated simply
-for covering more of the map). Normalising by the buffered area -- the region
-that actually collects points -- rather than the raw polygon keeps tiny unit
-postcodes from being over-inflated by the fixed buffer-ring floor. NOTE: this is
-an incident *density of the surrounding streets*, not a per-resident risk --
-zero-resident commercial centres (Soho, retail parks) legitimately rank high.
+One figure is produced for every postcode and crime type, averaged over the
+**last 7 years** and the **last 2 years**:
+
+* the **average annual incident count** -- ``sum(counts in covered window) * 12 /
+ covered_months`` -- the raw, absolute number of recorded incidents per year,
+ with no per-area or per-capita normalisation. A covered postcode with no
+ incidents of a type gets ``0``; a postcode whose force never published in the
+ window, or whose geometry is unusable, gets a *null* (unknown, never a zero).
+
+This figure is what the server exposes as the filterable crime feature -- the
+headline metric in the right pane. (An earlier version divided it by an ambient
+daytime population to get a per-1,000-people rate; that was hard to read, so the
+absolute per-year count is now used directly.)
**Force-coverage calendar.** police.uk has multi-year publication gaps for whole
forces (Greater Manchester has published nothing between 2019-07 and the present
@@ -29,28 +33,25 @@ computed against the months the postcode's own force actually published:
matched it (BTP, which reports nationwide, is excluded from the vote);
postcodes with no incidents inherit their outcode's majority force, then the
national modal force.
-* The headline ``"{type} (avg/yr)"`` is the POOLED annualised rate over the
- force's covered months: ``sum(counts in covered years) * 12 / covered_months``.
- Years in which the force published nothing contribute neither incidents nor
- months, so a coverage gap no longer reads as a low-crime period. (Pooling over
- covered months also fixes the old "divide by years-with-incidents" headline,
- which inflated sporadic categories by up to ~15x.)
-* The by-year series only emits bars for years with at least
- ``min_bar_months`` covered months (default 6): annualising a single observed
- month x12 produced misleading spikes. Each bar is scaled by the force's
- covered months in that year, not the global month calendar.
+* A window's average pools the counts over the force's *covered* months inside
+ that window and annualises by those months, so a coverage gap shrinks the data
+ rather than reading as a low-crime dip.
+* The by-year series only emits bars for years with at least ``min_bar_months``
+ covered months (default 6).
* ``covered_years`` (list[struct{year, months}]) is written for every postcode
- so the server can tell "covered, zero crime" (year listed, no bar) from "no
- data" (year absent) instead of charting gaps as zeros.
+ so the server can tell "covered, zero crime" from "no data".
* Postcodes whose boundary buffer is unusable (broken geometry) get null
- headline columns and an empty ``covered_years`` -- unknown, not zero.
+ figures and an empty ``covered_years`` -- unknown, not zero.
-Outputs mirror the old LSOA transform's shape but are keyed on ``postcode``:
+Outputs, all keyed on ``postcode``:
-* ``crime_by_postcode.parquet`` -- ``postcode`` + ``"{type} (avg/yr)"`` columns.
-* ``crime_by_postcode_by_year.parquet`` -- one row per postcode: ``postcode`` +
- ``covered_years`` + nested ``"{type} (by year)"`` ``list[struct{year, count}]``
- columns, with Serious/Minor rollups.
+* ``crime_by_postcode.parquet`` -- ``"{type} (/yr, 7y)"`` / ``"{type} (/yr, 2y)"``
+ average-annual-count columns (the filterable crime features).
+* ``crime_by_postcode_by_year.parquet`` -- ``covered_years`` + nested
+ ``"{type} (by year)"`` ``list[struct{year, count}]`` per-year raw counts.
+* ``crime_records.parquet`` -- one row per counted incident over the last 7
+ years (``postcode`` + month/type/location/outcome/lat/lon), sorted by
+ postcode so the server can slice each postcode's incidents directly.
Caveat: police.uk coordinates are snapped to a fixed set of anonymous "map
points", not true locations, and a share of rows have no coordinate at all
@@ -63,6 +64,7 @@ from __future__ import annotations
import argparse
import re
import sys
+import tempfile
from pathlib import Path
import numpy as np
@@ -80,22 +82,36 @@ from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
# Serious types first so column order is stable and self-documenting.
ALL_CRIME_TYPES: tuple[str, ...] = SERIOUS_CRIME_TYPES + MINOR_CRIME_TYPES
+# Output type axis = the 14 leaf types plus the two rollups, in that order.
+ROLLUP_TYPES: tuple[str, ...] = ("Serious crime", "Minor crime")
+ALL_OUTPUT_TYPES: tuple[str, ...] = ALL_CRIME_TYPES + ROLLUP_TYPES
-DEFAULT_BUFFER_M = 100.0
+DEFAULT_BUFFER_M = 50.0
MONTH_DIR_RE = re.compile(r"^\d{4}-\d{2}$")
STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
+# Trailing-window definitions, in (label, years) form. Each window's average is
+# pooled over the force's covered months inside the window; one average-annual-
+# count column (`(/yr, )`) — the filterable crime feature — is emitted per
+# window.
+WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))
+
+# Per-incident records cover the longest window's calendar years so the
+# "individual crimes" list reconciles exactly with the headline counts: every
+# record is an incident inside that window and vice versa. The headline counts
+# are pooled per calendar year, so the records window must be calendar-year
+# aligned too -- a rolling month span (e.g. a fixed 84 months) would include
+# incidents from a year the headline excludes when the latest month is mid-year.
+RECORDS_WINDOW_YEARS = max(win_years for _, win_years in WINDOWS)
+
# Minimum covered months for a year to get a by-year chart bar (and to be
# listed in `covered_years`). Annualising fewer observed months (x12 from a
-# single month at the worst) produces bars dominated by noise, and the first
-# (2010: one month) and current partial year would otherwise always chart as
-# spikes/dips. Six months keeps the annualisation factor <= 2.
+# single month at the worst) produces bars dominated by noise. Six months keeps
+# the annualisation factor <= 2.
MIN_BAR_MONTHS = 6
# Forces that report nationwide rather than policing a territory. They never
-# define a postcode's home force (their publication calendar says nothing about
-# whether the *territorial* force covering the postcode published), but their
-# incidents still count toward whichever postcodes they fall in.
+# define a postcode's home force, but their incidents still count.
NON_TERRITORIAL_FORCES = frozenset({"btp"})
COVERAGE_COLUMN = "covered_years"
@@ -106,8 +122,14 @@ LON_BOUNDS = (-9.5, 2.5)
LAT_BOUNDS = (49.0, 61.5)
# Read CSVs in chunks of files to bound peak memory while keeping the STRtree
-# query vectorised over a useful number of points.
-_CSV_BATCH = 64
+# query vectorised over a useful number of points. Kept modest because the
+# in-window batches also materialise the per-incident record strings.
+_CSV_BATCH = 32
+
+
+def crime_column(type_name: str, window: str) -> str:
+ """Filterable average-annual-count column, e.g. ``"Burglary (/yr, 7y)"``."""
+ return f"{type_name} (/yr, {window})"
def _force_calendar(
@@ -115,12 +137,9 @@ def _force_calendar(
) -> tuple[list[int], list[str], np.ndarray]:
"""Derive the per-force publication calendar from the CSV paths.
- Each police.uk file lives under ``{crime_dir}/{YYYY-MM}/{YYYY-MM}-{force}-
- street.csv`` and holds that force's incidents for that month, so file
- presence IS the coverage signal: a (force, month) with no file published
- nothing. Returns the sorted distinct years, the force slugs (sorted), and
- ``months_in_year_force`` of shape (n_forces, n_years) -- how many months
- each force published in each year.
+ File presence IS the coverage signal: a (force, month) with no file
+ published nothing. Returns the sorted distinct years, the force slugs
+ (sorted), and ``months_in_year_force`` of shape (n_forces, n_years).
"""
month_force: set[tuple[str, str]] = set()
for path in csvs:
@@ -142,9 +161,6 @@ def _force_calendar(
for month, force in month_force:
months_in_year_force[force_to_idx[force], year_to_idx[int(month[:4])]] += 1
- # Surface coverage gaps loudly: any territorial force missing months inside
- # the global publication window is exactly the data hole the coverage
- # masking exists for.
all_months = {month for month, _ in month_force}
for force in forces:
published = {m for m, f in month_force if f == force}
@@ -159,22 +175,25 @@ def _force_calendar(
def _build_tree(
polygons: np.ndarray, buffer_m: float
-) -> tuple[np.ndarray, shapely.STRtree]:
+) -> tuple[shapely.STRtree, np.ndarray]:
"""Buffer postcode polygons outward by ``buffer_m`` and index them.
- Buffer index == postcode index. Geometries that fail to buffer are replaced
- with an empty polygon so the index stays aligned; they simply never match.
+ Buffer index == postcode index. Returns the STRtree and a ``usable`` boolean
+ mask: geometries that fail to buffer are replaced with an empty polygon (so
+ the index stays aligned and they never match) and marked unusable, so their
+ crime picture is reported as unknown rather than zero.
"""
buffers = shapely.buffer(polygons, buffer_m, quad_segs=8)
broken = shapely.is_missing(buffers) | ~shapely.is_valid(buffers)
if broken.any():
print(f" {int(broken.sum()):,} postcode buffers unusable; left empty")
buffers[broken] = shapely.from_wkt("POLYGON EMPTY")
- return buffers, shapely.STRtree(buffers)
+ return shapely.STRtree(buffers), ~broken
def _accumulate_counts(
csvs: list[Path],
+ postcodes: np.ndarray,
tree: shapely.STRtree,
type_to_idx: dict[str, int],
year_to_idx: dict[int, int],
@@ -182,35 +201,49 @@ def _accumulate_counts(
transformer: Transformer,
counts: np.ndarray,
force_votes: np.ndarray,
+ records_shard_dir: Path | None,
+ records_min_ym: int,
) -> None:
"""Stream the crime CSVs, counting points-in-buffer per (postcode, type, year).
- Also accumulates ``force_votes`` (n_postcodes, n_forces): how many matched
- incidents each force's files contributed to each postcode, which later
- elects the postcode's home force for the coverage calendar.
+ Also accumulates ``force_votes`` (n_postcodes, n_forces) for the home-force
+ election and, when ``records_shard_dir`` is set, writes one parquet shard per
+ batch holding every counted (incident, postcode) pair whose month is within
+ the records window (month index >= ``records_min_ym``).
"""
+ # Type overrides only for the columns we ever read; LSOA is not stored.
schema = {
"Longitude": pl.Float64,
"Latitude": pl.Float64,
"Month": pl.Utf8,
"Crime type": pl.Utf8,
+ "Location": pl.Utf8,
+ "Last outcome category": pl.Utf8,
}
years = list(year_to_idx)
total_points = 0
total_matches = 0
total_dropped = 0
+ total_records = 0
unknown_type_counts: dict[str, int] = {}
for start in range(0, len(csvs), _CSV_BATCH):
batch = csvs[start : start + _CSV_BATCH]
- # The source file identifies the publishing force (police.uk has no
- # force column with consistent naming); map each path back to its
- # force index for the home-force vote.
path_to_fidx = {}
+ batch_max_ym = -1
for path in batch:
m = STREET_CSV_NAME_RE.fullmatch(path.name)
- if m is not None and m.group(2) in force_to_idx:
- path_to_fidx[str(path)] = force_to_idx[m.group(2)]
+ if m is not None:
+ ym = m.group(1)
+ batch_max_ym = max(batch_max_ym, int(ym[:4]) * 12 + int(ym[5:7]) - 1)
+ if m.group(2) in force_to_idx:
+ path_to_fidx[str(path)] = force_to_idx[m.group(2)]
+ # The per-incident record strings (Location, outcome) are by far the
+ # heaviest columns; read them only for batches that fall inside the
+ # records window, so the ~50% of pre-window months cost nothing extra.
+ want_records = records_shard_dir is not None and batch_max_ym >= records_min_ym
+ record_cols = ["Location", "Last outcome category"] if want_records else []
+
frame = (
pl.scan_csv(
batch,
@@ -218,12 +251,12 @@ def _accumulate_counts(
ignore_errors=True,
include_file_paths="_source_path",
)
- .select("Longitude", "Latitude", "Month", "Crime type", "_source_path")
- # strict=False: a single malformed Month drops only that row instead
- # of aborting the whole build (a non-numeric year becomes null and is
- # filtered out by the year membership check below).
+ .select(
+ "Longitude", "Latitude", "Month", "Crime type", *record_cols, "_source_path"
+ )
.with_columns(
- pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year")
+ pl.col("Month").str.slice(0, 4).cast(pl.Int32, strict=False).alias("year"),
+ pl.col("Month").str.slice(5, 2).cast(pl.Int32, strict=False).alias("_mm"),
)
.filter(
pl.col("Longitude").is_not_null()
@@ -234,14 +267,11 @@ def _accumulate_counts(
& (pl.col("Crime type") != "")
& pl.col("year").is_in(years)
)
- # Canonicalise legacy pre-2014 crime-type names ("Violent crime",
- # "Public disorder and weapons") to their current equivalents before
- # indexing, so ~1.9M historical incidents are counted instead of
- # dropped. `.replace` leaves current types unchanged.
+ # year*12 + (month-1): an integer month index for window filtering.
+ .with_columns(
+ (pl.col("year") * 12 + (pl.col("_mm").fill_null(1) - 1)).alias("month_index")
+ )
.with_columns(pl.col("Crime type").replace(LEGACY_CRIME_TYPE_ALIASES))
- # Map crime types to indices with default=None so an unrecognised
- # type yields a null index we can *report* rather than silently drop
- # (the legacy LSOA path surfaced unknown types via its dynamic pivot).
.with_columns(
pl.col("Crime type")
.replace_strict(type_to_idx, default=None, return_dtype=pl.Int32)
@@ -253,7 +283,16 @@ def _accumulate_counts(
.replace_strict(path_to_fidx, default=-1, return_dtype=pl.Int32)
.alias("fidx"),
)
- .select("Longitude", "Latitude", "Crime type", "tidx", "yidx", "fidx")
+ .select(
+ "Longitude",
+ "Latitude",
+ "Crime type",
+ *record_cols,
+ "month_index",
+ "tidx",
+ "yidx",
+ "fidx",
+ )
.collect(engine="streaming")
)
@@ -273,19 +312,15 @@ def _accumulate_counts(
tidx = frame["tidx"].to_numpy()
yidx = frame["yidx"].to_numpy()
fidx = frame["fidx"].to_numpy()
+ month_index = frame["month_index"].to_numpy()
x, y = transformer.transform(lon, lat)
finite = np.isfinite(x) & np.isfinite(y)
total_dropped += int((~finite).sum())
if not finite.any():
continue
- x, y, tidx, yidx, fidx = (
- x[finite],
- y[finite],
- tidx[finite],
- yidx[finite],
- fidx[finite],
- )
+ x, y = x[finite], y[finite]
+ tidx, yidx, fidx = tidx[finite], yidx[finite], fidx[finite]
total_points += x.size
points = shapely.points(x, y)
@@ -306,9 +341,26 @@ def _accumulate_counts(
)
total_matches += point_index.size
+ if want_records:
+ total_records += _write_record_shard(
+ records_shard_dir,
+ start,
+ postcodes,
+ point_index,
+ postcode_index,
+ month_index[finite],
+ frame["Crime type"].to_numpy()[finite],
+ frame["Location"].to_numpy()[finite],
+ frame["Last outcome category"].to_numpy()[finite],
+ lon[finite],
+ lat[finite],
+ records_min_ym,
+ )
+
print(
f" files {start + len(batch):,}/{len(csvs):,}: "
f"{total_points:,} located points, {total_matches:,} postcode matches"
+ + (f", {total_records:,} records" if records_shard_dir is not None else "")
)
if total_dropped:
@@ -329,19 +381,65 @@ def _accumulate_counts(
)
+def _write_record_shard(
+ records_shard_dir: Path,
+ start: int,
+ postcodes: np.ndarray,
+ point_index: np.ndarray,
+ postcode_index: np.ndarray,
+ month_index: np.ndarray,
+ crime_type: np.ndarray,
+ location: np.ndarray,
+ outcome: np.ndarray,
+ lon: np.ndarray,
+ lat: np.ndarray,
+ records_min_ym: int,
+) -> int:
+ """Write one parquet shard of (incident, postcode) records for this batch.
+
+ Each matched pair becomes a row -- the same multiplicity as the count -- so a
+ postcode's records are exactly the incidents that made up its counts. Only
+ incidents within the records window (month index >= ``records_min_ym``) are
+ kept. Returns the number of rows written.
+ """
+ mi = month_index[point_index]
+ keep = mi >= records_min_ym
+ if not keep.any():
+ return 0
+ pidx = point_index[keep]
+ # Build the string columns from Python lists (.tolist()) rather than numpy
+ # object arrays: an all-null Location/outcome slice is an all-None object
+ # array that polars cannot cast to String, whereas a list of str|None infers
+ # a nullable String column cleanly.
+ shard = pl.DataFrame(
+ {
+ "postcode": postcodes[postcode_index[keep]].astype(str),
+ "month_index": mi[keep].astype(np.int32),
+ "crime_type": crime_type[pidx].astype(str),
+ "location": location[pidx].tolist(),
+ "outcome": outcome[pidx].tolist(),
+ "lat": lat[pidx].astype(np.float32),
+ "lon": lon[pidx].astype(np.float32),
+ },
+ schema_overrides={
+ "postcode": pl.String,
+ "crime_type": pl.String,
+ "location": pl.String,
+ "outcome": pl.String,
+ },
+ )
+ shard.write_parquet(
+ records_shard_dir / f"{start:08d}.parquet", compression="zstd"
+ )
+ return shard.height
+
+
def _assign_home_force(
postcodes: np.ndarray,
force_votes: np.ndarray,
forces: list[str],
) -> np.ndarray:
- """Elect each postcode's home (territorial) force.
-
- Majority vote of matched incidents per publishing force; non-territorial
- forces (BTP) are excluded from the vote because their calendar says nothing
- about local coverage. Postcodes with no votes (no incidents ever, or
- BTP-only) inherit the majority force of their outcode, then the national
- modal force, so every postcode gets a coverage calendar.
- """
+ """Elect each postcode's home (territorial) force by majority incident vote."""
votes = force_votes.astype(np.int64, copy=True)
for idx, force in enumerate(forces):
if force in NON_TERRITORIAL_FORCES:
@@ -354,18 +452,15 @@ def _assign_home_force(
if not has_vote.any():
raise ValueError("No incidents matched any postcode; cannot assign forces")
- # Outcode-majority fallback for postcodes with no (territorial) incidents.
outcodes = np.array([pc.split(" ")[0] for pc in postcodes], dtype=object)
- national_modal = int(
- np.bincount(home[has_vote], minlength=len(forces)).argmax()
- )
+ national_modal = int(np.bincount(home[has_vote], minlength=len(forces)).argmax())
if (~has_vote).any():
outcode_modal: dict[str, int] = {}
voted_outcodes = outcodes[has_vote]
voted_home = home[has_vote]
for oc in np.unique(voted_outcodes):
- counts = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
- outcode_modal[oc] = int(counts.argmax())
+ tally = np.bincount(voted_home[voted_outcodes == oc], minlength=len(forces))
+ outcode_modal[oc] = int(tally.argmax())
fallback = np.array(
[outcode_modal.get(oc, national_modal) for oc in outcodes[~has_vote]],
dtype=np.int32,
@@ -379,10 +474,82 @@ def _assign_home_force(
return home
+def _window_annualised(
+ counts: np.ndarray,
+ months_in_year_force: np.ndarray,
+ home_fidx: np.ndarray,
+ usable: np.ndarray,
+ year_mask: np.ndarray,
+) -> np.ndarray:
+ """Raw annualised incidents/yr per (postcode, type) over a window of years.
+
+ For each force, the count is pooled over the force's covered months that fall
+ inside ``year_mask`` and annualised by those covered months. A covered
+ postcode with no incidents of a type gets 0; a postcode whose force never
+ published in the window, or whose geometry is unusable, gets NaN (unknown).
+ """
+ n_pc, n_types = counts.shape[0], counts.shape[1]
+ avg = np.full((n_pc, n_types), np.nan, dtype=np.float64)
+ for f in range(months_in_year_force.shape[0]):
+ sel = home_fidx == f
+ if not sel.any():
+ continue
+ cov_months = months_in_year_force[f].astype(np.float64) * year_mask
+ denom = cov_months.sum()
+ if denom <= 0:
+ continue # force published nothing in this window; stays NaN
+ window_years = cov_months > 0
+ pooled = counts[sel][:, :, window_years].sum(axis=2, dtype=np.float64)
+ avg[sel] = pooled * 12.0 / denom
+ avg[~usable] = np.nan
+ return avg
+
+
+def _append_rollups(avg14: np.ndarray) -> np.ndarray:
+ """Append Serious/Minor rollup columns (sum of components) -> (n_pc, 16)."""
+ serious_idx = [ALL_CRIME_TYPES.index(t) for t in SERIOUS_CRIME_TYPES]
+ minor_idx = [ALL_CRIME_TYPES.index(t) for t in MINOR_CRIME_TYPES]
+ serious = avg14[:, serious_idx].sum(axis=1)
+ minor = avg14[:, minor_idx].sum(axis=1)
+ return np.column_stack([avg14, serious, minor])
+
+
+def _write_crime_table(
+ postcodes: np.ndarray,
+ raw_by_window: dict[str, np.ndarray],
+ output_path: Path,
+) -> None:
+ """Write the average-annual-count parquet (the filterable crime features).
+
+ ``raw_by_window[label]`` is the (n_postcodes, 16) average annual incident
+ count for that window (14 leaf types + 2 rollups). Each value is the raw,
+ absolute incidents/yr; a postcode with no usable data for a window keeps NaN
+ (written as null).
+ """
+ data: dict[str, np.ndarray] = {"postcode": postcodes}
+ for label, _years in WINDOWS:
+ counts = np.round(raw_by_window[label], 1).astype(np.float32)
+ for type_idx, name in enumerate(ALL_OUTPUT_TYPES):
+ data[crime_column(name, label)] = counts[:, type_idx]
+
+ _write_nan_aware(data, output_path, "postcode crime average annual counts")
+
+
+def _write_nan_aware(
+ data: dict[str, np.ndarray], output_path: Path, label: str
+) -> None:
+ frame = pl.DataFrame(data)
+ value_cols = [c for c in frame.columns if c != "postcode"]
+ frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ frame.write_parquet(output_path, compression="zstd")
+ print(f"Wrote {label}: {output_path} {frame.shape}")
+
+
def _rollup_long(
long: pl.DataFrame, types: tuple[str, ...], rollup_name: str
) -> pl.DataFrame:
- """Sum per-year annualised counts across ``types`` into a single rollup."""
+ """Sum per-year counts across ``types`` into a single rollup."""
return (
long.filter(pl.col("Crime type").is_in(list(types)))
.group_by("postcode", "year")
@@ -392,108 +559,29 @@ def _rollup_long(
)
-def _write_avg_yr(
- postcodes: np.ndarray,
- counts: np.ndarray,
- months_in_year_force: np.ndarray,
- home_fidx: np.ndarray,
- norm: np.ndarray,
- output_path: Path,
-) -> None:
- """Write ``postcode`` + ``"{type} (avg/yr)"`` density-normalised averages.
-
- The headline is the POOLED annualised rate over the home force's covered
- months: ``sum(counts in covered years) * 12 / covered_months``. Years the
- force published nothing contribute neither incidents nor months, so a
- coverage gap (e.g. Greater Manchester 2019-07 onwards) is excluded instead
- of read as zero crime. Pooling over the full covered window -- rather than
- averaging only over years a type happened to occur -- is what keeps a
- single robbery-year from printing as a perennial robbery rate. Each
- postcode's value is then multiplied by ``norm`` (median_area / buffered
- catchment area) so the metric is a density rather than a footprint-inflated
- raw count; postcodes with unusable geometry (norm == 0) are null, not 0.
- """
- n_postcodes, n_types = counts.shape[0], counts.shape[1]
- avg = np.full((n_postcodes, n_types), np.nan, dtype=np.float64)
- for f in range(months_in_year_force.shape[0]):
- sel = home_fidx == f
- if not sel.any():
- continue
- cov_months = months_in_year_force[f].astype(np.float64)
- denom = cov_months.sum()
- if denom <= 0:
- continue # force never published; stays null
- covered_years = cov_months > 0
- pooled = counts[sel][:, :, covered_years].sum(axis=2, dtype=np.float64)
- avg[sel] = pooled * 12.0 / denom
-
- avg *= norm[:, None]
- avg[norm <= 0] = np.nan # unusable geometry: unknown, not zero
- avg = np.round(avg, 1).astype(np.float32)
-
- data: dict[str, np.ndarray] = {"postcode": postcodes}
- for type_idx, name in enumerate(ALL_CRIME_TYPES):
- data[f"{name} (avg/yr)"] = avg[:, type_idx]
-
- # Serious/Minor rollup headlines = the exact SUM of their component (avg/yr)
- # columns, so each rollup always equals the sum of the parts shown beside it
- # and can never fall below one of its own components. All components share
- # the postcode's pooled covered-month denominator, so the sum is itself the
- # pooled rollup rate. Null components (unusable geometry) propagate to a
- # null rollup.
- for rollup_name, rollup_types in (
- ("Serious crime", SERIOUS_CRIME_TYPES),
- ("Minor crime", MINOR_CRIME_TYPES),
- ):
- rollup_idx = [ALL_CRIME_TYPES.index(name) for name in rollup_types]
- data[f"{rollup_name} (avg/yr)"] = np.round(
- avg[:, rollup_idx].sum(axis=1), 1
- ).astype(np.float32)
-
- frame = pl.DataFrame(data)
- value_cols = [c for c in frame.columns if c != "postcode"]
- frame = frame.with_columns(pl.col(c).fill_nan(None) for c in value_cols)
-
- output_path.parent.mkdir(parents=True, exist_ok=True)
- frame.write_parquet(output_path, compression="zstd")
- print(f"Wrote postcode crime averages: {output_path}")
-
-
def _write_by_year(
postcodes: np.ndarray,
counts: np.ndarray,
years: list[int],
months_in_year_force: np.ndarray,
home_fidx: np.ndarray,
- norm: np.ndarray,
+ usable: np.ndarray,
min_bar_months: int,
output_path: Path,
) -> None:
- """Write nested ``"{type} (by year)"`` series plus rollups and coverage.
+ """Write nested ``"{type} (by year)"`` raw-count series plus rollups + coverage.
A bar is only emitted for (postcode, year)s where the postcode's home force
- published at least ``min_bar_months`` months -- annualising a thinner year
- (x12 from a single month at the extreme) charts noise, and a force-gap year
- must chart as *no data*, not zero. Bars are scaled by the force's covered
- months in that year and area-normalised by the same ``norm`` factor as the
- headline so chart and headline stay mutually consistent.
-
- Every postcode gets a row (the output is dense) carrying ``covered_years``
- -- the list of {year, months} the home force published at least
- ``min_bar_months`` months -- so consumers can distinguish covered-but-
- crime-free years (year listed, no bar => genuine zero) from coverage gaps
- (year absent => unknown). Postcodes with unusable geometry get an empty
- coverage list: their crime picture is unknown.
+ published at least ``min_bar_months`` months. Bars are the raw annualised
+ count for that year (``count * 12 / covered_months``); unlike the headline
+ windows there is no per-capita normalisation here -- the chart shows incident
+ volume over time. Every postcode gets a ``covered_years`` row so consumers can
+ distinguish covered-but-crime-free years from coverage gaps.
"""
- # (n_postcodes, n_years): covered months of each postcode's home force.
cov_pc_year = months_in_year_force[home_fidx, :]
- usable = norm > 0
annual = np.round(
- counts.astype(np.float64)
- * 12.0
- / np.maximum(cov_pc_year[:, None, :], 1)
- * norm[:, None, None],
+ counts.astype(np.float64) * 12.0 / np.maximum(cov_pc_year[:, None, :], 1),
1,
)
bar_ok = (
@@ -506,9 +594,6 @@ def _write_by_year(
type_names = np.array(ALL_CRIME_TYPES, dtype=object)
year_values = np.array(years, dtype=np.int32)
- # Explicit schema: with full masking (e.g. every year below min_bar_months)
- # the fancy-indexed numpy object arrays are empty and polars would infer
- # Object columns, which breaks the rollup `is_in` below.
long = pl.DataFrame(
{
"postcode": postcodes[pc_i].astype(str),
@@ -532,8 +617,6 @@ def _write_by_year(
type_cols = [c for c in wide.columns if c != "postcode"]
wide = wide.rename({col: f"{col} (by year)" for col in type_cols})
- # Dense base: every postcode, with its home force's coverage calendar.
- # Built per force (there are ~45) and joined on the force index.
coverage_per_force: list[list[dict[str, int]]] = []
for f in range(months_in_year_force.shape[0]):
coverage_per_force.append(
@@ -562,7 +645,6 @@ def _write_by_year(
dense = (
base.join(coverage_frame, on="_fidx", how="left")
.with_columns(
- # Unusable geometry: empty coverage -- the crime picture is unknown.
pl.when(pl.col("_usable"))
.then(pl.col(COVERAGE_COLUMN))
.otherwise(pl.col(COVERAGE_COLUMN).list.head(0))
@@ -577,11 +659,47 @@ def _write_by_year(
print(f"Wrote postcode crime by-year series: {output_path} {wide.shape}")
+def _finalize_records(
+ records_shard_dir: Path, output_path: Path
+) -> None:
+ """Concatenate the per-batch record shards into one postcode-sorted parquet.
+
+ Sorting by postcode lets the server build a contiguous per-postcode slice
+ index. The sort runs on the streaming engine so it spills rather than holding
+ all ~40M rows in memory.
+ """
+ shards = sorted(records_shard_dir.glob("*.parquet"))
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ if not shards:
+ pl.DataFrame(
+ schema={
+ "postcode": pl.String,
+ "month_index": pl.Int32,
+ "crime_type": pl.String,
+ "location": pl.String,
+ "outcome": pl.String,
+ "lat": pl.Float32,
+ "lon": pl.Float32,
+ }
+ ).write_parquet(output_path, compression="zstd")
+ print(f"Wrote crime records: {output_path} (empty)")
+ return
+
+ (
+ pl.scan_parquet(shards)
+ .sort("postcode")
+ .sink_parquet(output_path, compression="zstd")
+ )
+ n = pl.scan_parquet(output_path).select(pl.len()).collect().item()
+ print(f"Wrote crime records: {output_path} ({n:,} rows)")
+
+
def transform_crime_spatial(
crime_dir: Path,
boundaries_dir: Path,
output_path: Path,
by_year_output_path: Path,
+ records_output_path: Path,
buffer_m: float = DEFAULT_BUFFER_M,
max_postcodes: int | None = None,
max_files: int | None = None,
@@ -594,6 +712,10 @@ def transform_crime_spatial(
csvs = csvs[:max_files]
years, forces, months_in_year_force = _force_calendar(csvs)
+ latest_year = years[-1]
+ # Records cover the longest window's calendar years (January of its earliest
+ # year onward), so they reconcile exactly with the calendar-year headline.
+ records_min_ym = (latest_year - (RECORDS_WINDOW_YEARS - 1)) * 12
print(
f"Found {len(csvs):,} street crime CSVs across {len(forces)} forces "
f"({years[0]}-{years[-1]})"
@@ -601,27 +723,10 @@ def transform_crime_spatial(
)
postcodes, polygons = load_postcode_polygons(boundaries_dir, max_postcodes)
+ postcodes = np.asarray(postcodes)
print(f"Buffering {len(postcodes):,} postcode polygons by {buffer_m:g}m...")
- buffers, tree = _build_tree(polygons, buffer_m)
-
- # Area-normalisation factor (median_area / catchment_area): divides out the
- # size of each postcode's catchment so the count measures crime density, not
- # how much ground the buffer sweeps. We normalise by the *buffered* area --
- # the region that actually collects points -- rather than the raw polygon, so
- # a tiny unit postcode isn't over-inflated by the fixed buffer-ring floor.
- # Buffers are in EPSG:27700, so shapely.area is in m^2.
- areas = shapely.area(buffers).astype(np.float64)
- usable_area = np.isfinite(areas) & (areas > 0)
- if not usable_area.any():
- raise ValueError("No postcode buffers have a positive area to normalise by")
- median_area = float(np.median(areas[usable_area]))
- norm = np.zeros(len(postcodes), dtype=np.float64)
- norm[usable_area] = median_area / areas[usable_area]
- print(
- f"Area-normalising to median catchment area {median_area:,.0f} m^2 "
- f"({int(usable_area.sum()):,}/{len(areas):,} postcodes have usable area)"
- )
+ tree, usable = _build_tree(polygons, buffer_m)
type_to_idx = {name: idx for idx, name in enumerate(ALL_CRIME_TYPES)}
year_to_idx = {year: idx for idx, year in enumerate(years)}
@@ -630,25 +735,50 @@ def transform_crime_spatial(
force_votes = np.zeros((len(postcodes), len(forces)), dtype=np.int32)
transformer = Transformer.from_crs("EPSG:4326", "EPSG:27700", always_xy=True)
- _accumulate_counts(
- csvs, tree, type_to_idx, year_to_idx, force_to_idx, transformer, counts, force_votes
- )
- home_fidx = _assign_home_force(np.asarray(postcodes), force_votes, forces)
+ with tempfile.TemporaryDirectory(
+ prefix="crime_records_", dir=records_output_path.parent
+ ) as shard_dir_str:
+ shard_dir = Path(shard_dir_str)
+ _accumulate_counts(
+ csvs,
+ postcodes,
+ tree,
+ type_to_idx,
+ year_to_idx,
+ force_to_idx,
+ transformer,
+ counts,
+ force_votes,
+ shard_dir,
+ records_min_ym,
+ )
- _write_avg_yr(
- postcodes, counts, months_in_year_force, home_fidx, norm, output_path
- )
- _write_by_year(
- postcodes,
- counts,
- years,
- months_in_year_force,
- home_fidx,
- norm,
- min_bar_months,
- by_year_output_path,
- )
+ home_fidx = _assign_home_force(postcodes, force_votes, forces)
+
+ # Per-window raw annualised averages (14 leaf types + Serious/Minor).
+ raw_by_window: dict[str, np.ndarray] = {}
+ for label, win_years in WINDOWS:
+ year_mask = np.array(
+ [1.0 if y > latest_year - win_years else 0.0 for y in years]
+ )
+ avg14 = _window_annualised(
+ counts, months_in_year_force, home_fidx, usable, year_mask
+ )
+ raw_by_window[label] = _append_rollups(avg14)
+
+ _write_crime_table(postcodes, raw_by_window, output_path)
+ _write_by_year(
+ postcodes,
+ counts,
+ years,
+ months_in_year_force,
+ home_fidx,
+ usable,
+ min_bar_months,
+ by_year_output_path,
+ )
+ _finalize_records(shard_dir, records_output_path)
def main() -> None:
@@ -671,7 +801,7 @@ def main() -> None:
"--output",
type=Path,
required=True,
- help="Output parquet: postcode + '{type} (avg/yr)' columns",
+ help="Output parquet: postcode + '{type} (/yr, )' average-annual-count columns",
)
parser.add_argument(
"--output-by-year",
@@ -680,10 +810,10 @@ def main() -> None:
help="Output parquet: postcode + nested '{type} (by year)' columns",
)
parser.add_argument(
- "--buffer-m",
- type=float,
- default=DEFAULT_BUFFER_M,
- help="Outward buffer (metres) added to each postcode boundary",
+ "--output-records",
+ type=Path,
+ required=True,
+ help="Output parquet: one row per counted incident (last 7 years), postcode-sorted",
)
parser.add_argument(
"--max-postcodes",
@@ -705,15 +835,12 @@ def main() -> None:
)
args = parser.parse_args()
- if args.buffer_m <= 0:
- raise SystemExit("--buffer-m must be greater than zero")
-
transform_crime_spatial(
crime_dir=args.input,
boundaries_dir=args.boundaries,
output_path=args.output,
by_year_output_path=args.output_by_year,
- buffer_m=args.buffer_m,
+ records_output_path=args.output_records,
max_postcodes=args.max_postcodes,
max_files=args.max_files,
min_bar_months=args.min_bar_months,
diff --git a/pipeline/transform/join_price_estimates.py b/pipeline/transform/join_price_estimates.py
new file mode 100644
index 0000000..62d4ddd
--- /dev/null
+++ b/pipeline/transform/join_price_estimates.py
@@ -0,0 +1,149 @@
+"""Join the slim price estimates back onto properties.parquet.
+
+Price estimation runs on ``price_inputs.parquet`` (built by ``property_base``
+straight from epc_pp + arcgis, independently of merge's area features) and emits
+``price_estimates.parquet`` — the natural key (Postcode + coalesced address) plus
+``Estimated current price`` / ``Est. price per sqm``. This step joins those two
+columns onto properties.parquet to produce the file the server consumes.
+
+Why the natural key
+-------------------
+Estimates and properties are built by separate runs, so a positional row index
+would not line up. Instead both derive the key ``(Postcode, coalesce(register
+address, EPC address))`` — which is unique and non-null on the deduped dwelling
+universe (see ``property_base._dedupe_collapsed_properties``) and identical on
+both sides because both start from that same universe. So estimates map onto
+properties 1:1 regardless of row order.
+
+Re-running is safe: any pre-existing estimate columns are dropped first, and the
+join is keyed (not positional), so a second run reproduces the same result. The
+join refuses if any property has no estimate (the dwelling universes diverged,
+e.g. a stale price_inputs vs a newer epc_pp) rather than silently leaving prices
+null. Output is written to a temp file and atomically renamed.
+"""
+
+import argparse
+from pathlib import Path
+
+import polars as pl
+
+from pipeline.transform.price_estimation.utils import (
+ ESTIMATE_COLUMNS,
+ JOIN_ADDRESS,
+ JOIN_KEYS,
+ join_address_expr,
+)
+
+
+def join_estimates(properties: Path, estimates_path: Path) -> int:
+ """Augment ``properties`` in place with the estimate columns; return rows.
+
+ Joins the slim estimates onto properties by the natural key and atomically
+ replaces properties.parquet. Idempotent: any estimate columns already on the
+ file are dropped first.
+ """
+ estimates = pl.scan_parquet(estimates_path)
+ est_cols = estimates.collect_schema().names()
+ missing = [c for c in (*JOIN_KEYS, *ESTIMATE_COLUMNS) if c not in est_cols]
+ if missing:
+ raise ValueError(f"{estimates_path}: missing columns {missing}")
+
+ stats = estimates.select(
+ n=pl.len(), unique=pl.struct(JOIN_KEYS).n_unique()
+ ).collect(engine="streaming")
+ n_estimates, n_unique = stats["n"][0], stats["unique"][0]
+ if n_unique != n_estimates:
+ raise ValueError(
+ f"{estimates_path}: natural key {JOIN_KEYS} is not unique "
+ f"({n_estimates - n_unique:,} duplicate rows)"
+ )
+
+ n_properties = pl.scan_parquet(properties).select(pl.len()).collect().item()
+
+ # Drop any estimate columns already present (idempotent re-run) and attach the
+ # coalesced-address half of the natural key.
+ properties_keyed = (
+ pl.scan_parquet(properties)
+ .drop(ESTIMATE_COLUMNS, strict=False)
+ .with_columns(join_address_expr())
+ )
+
+ # Every property must have an estimate: estimates and properties come from the
+ # same dwelling universe, so a gap means a stale/foreign price_inputs (e.g.
+ # built from a different epc_pp). Fail loudly instead of nulling prices.
+ #
+ # This assumes properties.parquet contains ONLY epc_pp-derived dwellings, which
+ # is true for the production merge output. Running merge with --actual-listings
+ # appends listing seed rows whose (Postcode, address) keys are absent from
+ # price_inputs (built straight from epc_pp), which would trip the guard below.
+ # Enabling listing integration on the primary output therefore requires
+ # price_inputs to include those seed rows too.
+ unmatched = (
+ properties_keyed.select(JOIN_KEYS)
+ .join(estimates.select(JOIN_KEYS), on=JOIN_KEYS, how="anti")
+ .select(pl.len())
+ .collect(engine="streaming")
+ .item()
+ )
+ if unmatched:
+ raise ValueError(
+ f"{properties}: {unmatched:,} of {n_properties:,} properties have no "
+ "matching estimate; the price_inputs and properties dwelling universes "
+ "differ (regenerate price_inputs.parquet from the current epc_pp)."
+ )
+
+ # maintain_order="left" keeps properties in merge's row order; the unique key
+ # cannot fan the join out, so the row count is preserved.
+ result = properties_keyed.join(
+ estimates, on=JOIN_KEYS, how="left", maintain_order="left"
+ ).drop(JOIN_ADDRESS)
+
+ tmp = properties.with_name(properties.name + ".tmp")
+ result.sink_parquet(tmp)
+
+ written = pl.scan_parquet(tmp).select(pl.len()).collect().item()
+ if written != n_properties:
+ tmp.unlink(missing_ok=True)
+ raise ValueError(
+ f"{properties}: join changed the row count "
+ f"({n_properties:,} -> {written:,})"
+ )
+
+ tmp.replace(properties)
+ return written
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Join price_estimates.parquet onto properties.parquet"
+ )
+ parser.add_argument(
+ "--properties",
+ type=Path,
+ required=True,
+ help="properties.parquet (read, then overwritten with the estimate "
+ "columns joined in)",
+ )
+ parser.add_argument(
+ "--estimates",
+ type=Path,
+ required=True,
+ help="Slim price_estimates.parquet from price_estimation.estimate",
+ )
+ args = parser.parse_args()
+
+ written = join_estimates(args.properties, args.estimates)
+ size_mb = args.properties.stat().st_size / (1024 * 1024)
+ n_priced = (
+ pl.scan_parquet(args.properties)
+ .filter(pl.col("Estimated current price").is_not_null())
+ .select(pl.len())
+ .collect()
+ .item()
+ )
+ print(f"Wrote {args.properties} ({size_mb:.1f} MB)")
+ print(f" {written:,} rows, {n_priced:,} with an estimated current price")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pipeline/transform/merge.py b/pipeline/transform/merge.py
index a1d2007..acbfbae 100644
--- a/pipeline/transform/merge.py
+++ b/pipeline/transform/merge.py
@@ -29,10 +29,16 @@ from pipeline.utils.fuzzy_join import (
normalize_address_key,
normalize_postcode_key,
)
+from pipeline.transform.property_base import (
+ MIN_FLOOR_AREA_M2,
+ _active_english_postcode_area,
+ _filter_to_active_english_postcodes,
+ build_property_base,
+ property_type_expr,
+)
from pipeline.utils.normalize import drop_digit_tokens
from pipeline.utils.postcode_mapping import build_postcode_mapping
-MIN_FLOOR_AREA_M2 = 10
CONSERVATION_AREA_FEATURE = "Within conservation area"
# Named "Tree canopy" (not "Street tree") because the underlying density unions
# Forest Research TOW lone-tree/group crowns AND NFI woodland canopy, so a
@@ -77,23 +83,42 @@ _AREA_COLUMNS = [
"% Mixed",
"% White",
"% Other",
- # Crime
- "Anti-social behaviour (avg/yr)",
- "Violence and sexual offences (avg/yr)",
- "Criminal damage and arson (avg/yr)",
- "Burglary (avg/yr)",
- "Vehicle crime (avg/yr)",
- "Robbery (avg/yr)",
- "Other theft (avg/yr)",
- "Shoplifting (avg/yr)",
- "Drugs (avg/yr)",
- "Possession of weapons (avg/yr)",
- "Public order (avg/yr)",
- "Bicycle theft (avg/yr)",
- "Theft from the person (avg/yr)",
- "Other crime (avg/yr)",
- "Serious crime (avg/yr)",
- "Minor crime (avg/yr)",
+ # Crime — average annual recorded incident count (incidents/yr), 7-year and
+ # 2-year windows. These are the filterable crime features; the per-incident
+ # records live in a separate side table the server loads directly (it bypasses
+ # the merge).
+ "Anti-social behaviour (/yr, 7y)",
+ "Anti-social behaviour (/yr, 2y)",
+ "Violence and sexual offences (/yr, 7y)",
+ "Violence and sexual offences (/yr, 2y)",
+ "Criminal damage and arson (/yr, 7y)",
+ "Criminal damage and arson (/yr, 2y)",
+ "Burglary (/yr, 7y)",
+ "Burglary (/yr, 2y)",
+ "Vehicle crime (/yr, 7y)",
+ "Vehicle crime (/yr, 2y)",
+ "Robbery (/yr, 7y)",
+ "Robbery (/yr, 2y)",
+ "Other theft (/yr, 7y)",
+ "Other theft (/yr, 2y)",
+ "Shoplifting (/yr, 7y)",
+ "Shoplifting (/yr, 2y)",
+ "Drugs (/yr, 7y)",
+ "Drugs (/yr, 2y)",
+ "Possession of weapons (/yr, 7y)",
+ "Possession of weapons (/yr, 2y)",
+ "Public order (/yr, 7y)",
+ "Public order (/yr, 2y)",
+ "Bicycle theft (/yr, 7y)",
+ "Bicycle theft (/yr, 2y)",
+ "Theft from the person (/yr, 7y)",
+ "Theft from the person (/yr, 2y)",
+ "Other crime (/yr, 7y)",
+ "Other crime (/yr, 2y)",
+ "Serious crime (/yr, 7y)",
+ "Serious crime (/yr, 2y)",
+ "Minor crime (/yr, 7y)",
+ "Minor crime (/yr, 2y)",
# Amenities
"Number of restaurants within 2km",
"Number of grocery shops and supermarkets within 2km",
@@ -189,8 +214,6 @@ _FINAL_RENAME_COLUMNS = {
"outstanding_primary_catchments": "Outstanding primary school catchments",
"outstanding_secondary_catchments": "Outstanding secondary school catchments",
"max_download_speed": "Max available download speed (Mbps)",
- "serious_crime_avg_yr": "Serious crime (avg/yr)",
- "minor_crime_avg_yr": "Minor crime (avg/yr)",
"mean_monthly_rent": "Estimated monthly rent",
"floor_height": "Interior height (m)",
"was_council_house": "Former council house",
@@ -822,78 +845,6 @@ def _validate_property_postcodes(df: pl.DataFrame) -> None:
)
-def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
- """Return the supported postcode universe with geography join keys."""
- return (
- arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
- .filter(pl.col("doterm").is_null())
- .select(
- pl.col("pcds").alias("postcode"),
- "lat",
- pl.col("long").alias("lon"),
- "ctry25cd",
- pl.col("lsoa21cd").alias("lsoa21"),
- pl.col("oa21cd").alias("oa21"),
- pl.col("pcon24cd").alias("pcon"),
- )
- .drop_nulls(["postcode"])
- .unique(["postcode"])
- )
-
-
-def _remap_terminated_postcodes(
- wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
-) -> pl.LazyFrame:
- return (
- wide.join(
- postcode_mapping,
- left_on="postcode",
- right_on="old_postcode",
- how="left",
- )
- .with_columns(
- pl.coalesce("new_postcode", "postcode").alias("postcode"),
- )
- .drop("new_postcode")
- )
-
-
-def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
- """Keep one row per (postcode, address) — the most-recent transaction.
-
- The terminated-postcode remap can map two distinct postcodes onto one active
- successor, collapsing the same physical address onto a single
- (postcode, address) key with conflicting sale records. Keep the row with the
- latest date_of_transfer so the headline price/date reflect the most recent
- transaction; genuinely distinct addresses are untouched.
-
- The dedup key coalesces the price-paid address with the EPC address: EPC-only
- dwellings (never sold) have a null pp_address, so keying on pp_address alone
- would collapse EVERY EPC-only dwelling in a postcode onto one
- (postcode, null) key and silently drop all but one. Each dwelling's coalesced
- address is unique within its postcode (the EPC frame is deduped on
- address+postcode upstream), so the coalesced key keeps them distinct while
- leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
- a sale exists.
- """
- return (
- wide.with_columns(
- pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
- )
- .sort("date_of_transfer", descending=True, nulls_last=True)
- .unique(
- subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
- )
- .drop("_dedupe_address")
- )
-
-
-def _filter_to_active_english_postcodes(
- wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
-) -> pl.LazyFrame:
- return wide.join(active_postcodes, on="postcode", how="semi")
-
-
def _join_area_side_tables(
base: pl.LazyFrame,
*,
@@ -923,21 +874,15 @@ def _join_area_side_tables(
# joined on the same `lsoa21` key as ethnicity, education, IoD, and median age.
base = base.join(tenure, on="lsoa21", how="left")
- # Crime is counted spatially per postcode (incidents within 50m of the
- # postcode boundary), so it joins on postcode rather than LSOA. crime_spatial
- # precomputes the Serious/Minor headline rollups as the mean of the by-year
- # rollup bars; read those straight through (renamed to the internal columns
- # _finalize_merged_columns expects) rather than re-summing the per-type
- # avg/yr columns — summing divides each type by its OWN years-present and
- # overstates the rollup when types differ in coverage. A postcode absent from
- # the crime table keeps null rollups via the left join (no fabricated zero);
- # the per-type avg/yr columns pass through unchanged for display.
- base = base.join(crime, on="postcode", how="left").rename(
- {
- "Serious crime (avg/yr)": "serious_crime_avg_yr",
- "Minor crime (avg/yr)": "minor_crime_avg_yr",
- }
- )
+ # Crime is counted spatially per postcode (incidents within the boundary
+ # buffer), so it joins on postcode rather than LSOA. crime_spatial writes
+ # average-annual-count columns ("{type} (/yr, 7y|2y)"), including the
+ # Serious/Minor rollups (the exact sum of their components); all pass straight
+ # through to display/filtering. A postcode absent from the crime table keeps
+ # null values via the left join (no fabricated zero). The per-incident records
+ # are a separate side table the server loads directly, so it is not joined
+ # here.
+ base = base.join(crime, on="postcode", how="left")
base = base.join(median_age, on="lsoa21", how="left")
base = base.join(election, on="pcon", how="left")
@@ -2386,27 +2331,17 @@ def _build(
)
_validate_lad_source_coverage(iod_path, rental_prices_path)
- wide = pl.scan_parquet(epc_pp_path).filter(
- pl.col("total_floor_area").is_null()
- | (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
- )
-
- # Remap terminated postcodes to nearest active successor before filtering to
- # the supported active-English postcode universe. Historical properties from
- # terminated English postcodes are retained under their successor postcode.
- postcode_mapping = build_postcode_mapping(arcgis_path)
- wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
- # The remap can collapse two terminated postcodes onto one active successor,
- # duplicating a physical address's (postcode, pp_address) key; keep only the
- # most-recent transaction per address before the per-postcode joins.
- wide = _dedupe_collapsed_properties(wide)
+ # The dwelling universe — floor filter, terminated-postcode remap,
+ # collapse-dedupe, restrict to active English postcodes — is shared with
+ # price estimation so estimates line up 1:1 with these rows. See
+ # pipeline.transform.property_base.
+ wide = build_property_base(epc_pp_path, arcgis_path)
arcgis_raw = pl.scan_parquet(arcgis_path)
arcgis = _active_english_postcode_area(arcgis_raw)
active_postcodes = arcgis.select("postcode").unique()
active_postcode_count = (
active_postcodes.select(pl.len()).collect(engine="streaming").item()
)
- wide = _filter_to_active_english_postcodes(wide, active_postcodes)
if listed_buildings_path is not None:
active_postcodes_for_listed = (
@@ -2542,37 +2477,10 @@ def _build(
how="left",
)
- # Derive property_type: prefer EPC data, fall back to price-paid.
- # For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer detail.
- bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
- ["NO DATA!", "Not Recorded"]
- )
- has_epc = pl.col("epc_property_type").is_not_null()
- is_house = pl.col("epc_property_type") == "House"
- wide = wide.with_columns(
- pl.when(has_epc & is_house & ~bad_built_form)
- .then(pl.col("built_form"))
- .when(has_epc & is_house)
- .then(pl.col("pp_property_type"))
- .when(has_epc)
- .then(pl.col("epc_property_type"))
- .otherwise(pl.col("pp_property_type"))
- # Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
- # collapse terrace sub-types, and fold rare types into "Other"
- .replace(
- {
- "Flat": "Flats/Maisonettes",
- "Maisonette": "Flats/Maisonettes",
- "End-Terrace": "Terraced",
- "Mid-Terrace": "Terraced",
- "Enclosed End-Terrace": "Terraced",
- "Enclosed Mid-Terrace": "Terraced",
- "Bungalow": "Other",
- "Park home": "Other",
- }
- )
- .alias("property_type")
- )
+ # Derive property_type (EPC preferred, price-paid fallback, built_form for
+ # houses). Shared with price_inputs so the estimate uses the same type; see
+ # property_base.property_type_expr.
+ wide = wide.with_columns(property_type_expr().alias("property_type"))
wide = wide.with_columns(
pl.when(pl.col("duration") == "U")
diff --git a/pipeline/transform/postcode_boundaries/README.md b/pipeline/transform/postcode_boundaries/README.md
index 01d6181..93f393f 100644
--- a/pipeline/transform/postcode_boundaries/README.md
+++ b/pipeline/transform/postcode_boundaries/README.md
@@ -79,7 +79,9 @@ The output of `process_oa` is `list[(postcode, polygon)]` — the per-OA fragmen
**Fragment merging** (`output.py:merge_fragments`): Groups all fragments by postcode, unions them. If the result is a MultiPolygon (meaning the postcode has disconnected pieces — either from spanning OAs with a gap, or algorithm artifacts), applies a 5m buffer-then-unbuffer to close tiny gaps from floating-point mismatches at OA boundary edges. If still a MultiPolygon after that, keeps the largest part **plus any other part ≥ `_MIN_DETACHED_PART_AREA` (100 m²)** (`_keep_polygon_parts`); only sub-100 m² noise slivers are dropped. Keeping substantial detached parts matters because a postcode genuinely split across an OA seam (by a railway, river, or main road wider than the 5m buffer) would otherwise lose a chunk — measured at ~1.8% of merged area left as uncovered gaps (often 3000–5000 m² building blocks) before this change.
-**GeoJSON output** (`output.py:write_district_geojson`): Two passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
+**Greenspace subtraction is connectivity-preserving** (`greenspace.py:subtract_greenspace`): park/water polygons are subtracted from each postcode, but greenspace that *crosses* a postcode (a river, a strip of parkland, a golf course through a village) would otherwise split it into scattered pieces. When the subtraction disconnects a postcode, `_reconnect_split` re-adds the narrowest removed necks — a morphological closing (`_RECONNECT_BRIDGE_M`, 25 m) clipped to the original postcode footprint — so parts ≤ ~50 m apart stay joined by a thin bridge of the postcode's own land (no address moves); genuinely wide barriers stay subtracted and the postcode legitimately splits.
+
+**GeoJSON output** (`output.py:write_district_geojson`): three passes. Pass 1 converts every postcode from BNG to WGS84 (pyproj), simplifies with 1m tolerance (Douglas-Peucker), and snaps to 6 decimal places (~0.1m precision); multi-part postcodes become `MultiPolygon` (`to_wgs84_geojson_multi`, each part handled independently), single-part stay `Polygon`. The whole set is then made a **partition** (`_resolve_overlaps`): each postcode is trimmed by the union of its higher-priority overlapping neighbours, where **priority = ascending area** (smaller postcodes win contested ground). That single rule handles both seam overlap *and* containment — an enclosed postcode is always smaller than its container, so it keeps its area while the container gets a hole (the query uses both the `overlaps` and `contains` predicates, since `overlaps` alone excludes containment). This runs last, so nothing re-introduces overlap; a postcode that would be emptied keeps its original geometry, so no active postcode is dropped. Pass 2 **de-fragments** the partition (`_eliminate_small_detached_parts`): a detached part that is *both* small in absolute terms (< `_ELIM_ABS_MAX_M2`, 3000 m²) *and* a minor fraction (< `_ELIM_FRAC_MAX`, 15%) of its postcode is absorbed into the neighbouring postcode it shares the most boundary with — the classic GIS *eliminate*. This removes the Voronoi/INSPIRE/seam *scatter* that left ~1/3 of postcodes non-contiguous, while a genuine bisection (two substantial parts split by a river/railway) keeps both parts. The land is **reassigned**, never dropped, so the output stays a gapless partition and coverage is conserved; the largest part of every postcode is always retained, so no active postcode is dropped (a tiny neighbour-less sliver in removed greenspace is dropped, a larger isolated patch is kept). Pass 3 groups postcodes by district (the outward code, e.g. `SW1A` from `SW1A 1AA`), rounds coordinates to 6dp, and writes a `{district}.geojson` FeatureCollection. Each Feature has `postcodes` (formatted like `"SW1A 1AA"`) and `mapit_code` (no space: `"SW1A1AA"`) in its properties.
## Memory architecture
@@ -107,6 +109,7 @@ Key design choices:
2. **Every postcode that exists in the UPRN data gets a polygon** — unless all its UPRNs share coordinates with another postcode's UPRNs (handled by jitter) or it has zero UPRNs
3. **Postcode polygons never extend outside their OA(s)** — all geometry is clipped to OA boundaries
4. **A postcode split across an OA seam keeps all its substantial parts** — `merge_fragments` keeps every part ≥ 100 m² and the output is emitted as a `MultiPolygon` (the Rust server `postcodes.rs` and `loader.py` both parse MultiPolygon); only sub-100 m² noise slivers are dropped
+5. **Postcodes are contiguous unless genuinely split** — most non-contiguity is *scatter* (a unit drawn as many disconnected specks) from point-Voronoi over sparse/interleaved UPRNs, greenspace cutting across a unit, and overlap/seam slivers. Connectivity-preserving greenspace subtraction + the `_eliminate_small_detached_parts` de-fragmentation pass absorb that scatter into neighbours (coverage-conserving), cutting the share of multi-part postcodes roughly in half (~30% → ~14% measured on the worst rural/coastal districts) without dropping any postcode or leaving coverage gaps. Genuine bisections (river/railway/major road, or a detached part above the absolute+fraction thresholds) are preserved.
## Module structure
diff --git a/pipeline/transform/postcode_boundaries/greenspace.py b/pipeline/transform/postcode_boundaries/greenspace.py
index 93a0d7b..d961700 100644
--- a/pipeline/transform/postcode_boundaries/greenspace.py
+++ b/pipeline/transform/postcode_boundaries/greenspace.py
@@ -7,7 +7,7 @@ from shapely import make_valid, wkb
from shapely.geometry import MultiPolygon, Polygon
from shapely.strtree import STRtree
-from .geometry import safe_difference, safe_union
+from .geometry import _SNAP_GRID, _poly_valid, safe_difference, safe_intersection, safe_union
def load_greenspace(path: Path) -> tuple[STRtree, list]:
@@ -36,6 +36,42 @@ def load_greenspace(path: Path) -> tuple[STRtree, list]:
MAX_REMOVAL_FRACTION = 0.9 # Keep original if >90% would be removed
+# Greenspace that merely trims the edge of a postcode is fine, but greenspace
+# that CROSSES it (a river, a strip of parkland, a golf course running through a
+# village) splits the postcode into a MultiPolygon -- one the map then draws as
+# several disconnected pieces. When subtraction disconnects a postcode, re-add
+# the postcode's OWN removed land along the narrowest necks (a morphological
+# closing clipped to the original footprint) so the parts stay joined by a thin
+# bridge. Parts left more than ~2x this width apart (a genuinely wide barrier)
+# stay split. Because the bridge is the postcode's own land, no address moves and
+# only a thin sliver of green is kept back.
+_RECONNECT_BRIDGE_M = 25.0
+
+
+def _reconnect_split(
+ result: Polygon | MultiPolygon, postcode_geom: Polygon | MultiPolygon
+) -> Polygon | MultiPolygon:
+ """Re-join postcode parts that greenspace subtraction pulled apart by re-adding
+ the narrow removed necks (within the original postcode), leaving wide barriers
+ intact."""
+ if result.geom_type != "MultiPolygon":
+ return result
+ closed = result.buffer(_RECONNECT_BRIDGE_M).buffer(-_RECONNECT_BRIDGE_M)
+ if not closed.is_valid:
+ closed = make_valid(closed)
+ # The closing material that lies inside the original postcode but outside the
+ # subtraction result == the thin green necks linking the parts. The exact
+ # overlay path can leave line/point debris (coincident edges) that is
+ # zero-area but NOT is_empty; `_poly_valid` strips it to polygons only, so the
+ # is_empty guard works and the union can never return a GeometryCollection
+ # (which `to_wgs84_geojson_multi` would silently truncate to a single part).
+ bridges = _poly_valid(
+ safe_difference(safe_intersection(closed, postcode_geom), result), _SNAP_GRID
+ )
+ if bridges.is_empty:
+ return result
+ return _poly_valid(safe_union([result, bridges]), _SNAP_GRID)
+
def subtract_greenspace(
postcode_geom: Polygon | MultiPolygon,
@@ -48,6 +84,10 @@ def subtract_greenspace(
of intersecting greenspace from the postcode polygon. If subtraction
would remove >90% of the area, keeps the original (the postcode
genuinely covers that land, e.g. churchyards, riverside addresses).
+
+ If the subtraction disconnects the postcode (greenspace crossing it),
+ :func:`_reconnect_split` re-adds the narrowest removed necks so the postcode
+ stays a single piece rather than shipping as scattered fragments.
"""
candidate_idxs = tree.query(postcode_geom)
if len(candidate_idxs) == 0:
@@ -74,4 +114,4 @@ def subtract_greenspace(
if original_area > 0 and result.area / original_area < (1 - MAX_REMOVAL_FRACTION):
return postcode_geom
- return result
+ return _reconnect_split(result, postcode_geom)
diff --git a/pipeline/transform/postcode_boundaries/output.py b/pipeline/transform/postcode_boundaries/output.py
index 6f3707f..4d5aa58 100644
--- a/pipeline/transform/postcode_boundaries/output.py
+++ b/pipeline/transform/postcode_boundaries/output.py
@@ -1,4 +1,5 @@
import json
+import math
import shutil
from collections import defaultdict
from pathlib import Path
@@ -383,6 +384,164 @@ def _resolve_overlaps(
return [(pc, out[i]) for i, (pc, _) in enumerate(items)]
+# A detached (non-largest) part of a postcode is absorbed into the neighbouring
+# postcode it shares the most boundary with, when the part is BOTH small in
+# absolute terms AND a minor fraction of its postcode. This removes the
+# Voronoi/INSPIRE/overlap-seam SCATTER that otherwise left ~1/3 of postcodes
+# non-contiguous (a single unit drawn as many disconnected specks), while keeping
+# genuine splits: a postcode bisected by a river/railway into two SUBSTANTIAL
+# parts has both parts above these thresholds and is left alone. The land is
+# REASSIGNED to an adjacent postcode wherever a neighbour exists, so coverage is
+# conserved and the output stays a partition; only a tiny neighbour-less sliver
+# (< _ELIM_DROP_BELOW_M2, snap/overlap debris floating in removed greenspace) is
+# dropped, and a larger isolated part (a genuine detached hamlet) is kept. The
+# largest part of every postcode is always retained, so no postcode is dropped.
+_ELIM_ABS_MAX_M2 = 3000.0 # absolute size below which a minor part reads as scatter
+_ELIM_FRAC_MAX = 0.15 # ...and only when it is < this fraction of the postcode
+_ELIM_DROP_BELOW_M2 = 200.0 # a neighbour-less sliver this small is snap debris -> drop
+# WGS84-degree distances at UK latitudes (1e-6 deg ~ 0.11 m at ~53N)
+_ELIM_SHARED_EPS_DEG = 5e-7 # ~0.05 m: thin probe whose overlap area ~ shared-edge length
+# Candidate-gather + nearest-neighbour fallback radius, in degrees so it needs no
+# per-part reprojection. The metric reach is mildly anisotropic (~2.2 m N-S,
+# ~1.3-1.4 m E-W at England's latitudes); this only bounds the rare gapped-seam
+# fallback (the dominant border-sharing path is unaffected), so the approximation
+# is harmless. Gather and accept use the SAME radius, so there is no dead band.
+_ELIM_NEAREST_MAX_DEG = 2e-5
+_M_PER_DEG_LAT = 111_320.0
+_ELIM_ITERATIONS = 2
+
+
+def _approx_area_m2(geom_deg) -> float:
+ """Metric area of a WGS84 polygon via a latitude-scaled planar approximation.
+
+ Accurate to a few percent at the few-hundred-to-few-thousand m^2 scale these
+ thresholds work at, and far cheaper than a per-part CRS transform over the
+ full ~1.8M postcodes.
+ """
+ area_deg2 = geom_deg.area
+ if area_deg2 == 0.0:
+ return 0.0
+ centroid = geom_deg.centroid
+ if centroid.is_empty:
+ lat = (geom_deg.bounds[1] + geom_deg.bounds[3]) / 2
+ else:
+ lat = centroid.y
+ return area_deg2 * _M_PER_DEG_LAT * _M_PER_DEG_LAT * math.cos(math.radians(lat))
+
+
+def _geom_parts(geom):
+ return list(geom.geoms) if geom.geom_type == "MultiPolygon" else [geom]
+
+
+def _eliminate_small_detached_parts(
+ items: list[tuple[str, Polygon | MultiPolygon]],
+) -> list[tuple[str, Polygon | MultiPolygon]]:
+ """De-fragment the partition: absorb small detached parts into the neighbour
+ they border most (see the module note above).
+
+ Runs on the de-overlapped WGS84 geometries as the last shaping step. The
+ largest part of every postcode is always retained, so no active postcode is
+ dropped; parts are moved between postcodes (or, for tiny neighbour-less
+ slivers, dropped), so total covered area is conserved to within rounding.
+ """
+ geoms: dict[str, Polygon | MultiPolygon] = {pc: g for pc, g in items}
+ for _ in range(_ELIM_ITERATIONS):
+ recs: list[tuple[str, Polygon, float]] = []
+ total: dict[str, float] = defaultdict(float)
+ largest: dict[str, float] = defaultdict(float)
+ for pc, geom in geoms.items():
+ for part in _geom_parts(geom):
+ if part.is_empty:
+ continue
+ area = _approx_area_m2(part)
+ recs.append((pc, part, area))
+ total[pc] += area
+ if area > largest[pc]:
+ largest[pc] = area
+
+ tree = STRtree([part for _, part, _ in recs])
+ assign: dict[int, str | None] = {}
+ for i, (pc, part, area) in enumerate(recs):
+ if area >= largest[pc]:
+ continue # the largest part always stays -> a postcode never vanishes
+ if not (area < _ELIM_ABS_MAX_M2 and area < _ELIM_FRAC_MAX * total[pc]):
+ continue # substantial or major-fraction part = genuine split, keep
+
+ best_pc = None
+ best_score = 0.0
+ nearest_pc = None
+ nearest_d = float("inf")
+ # Gather candidates out to the nearest-fallback radius (not just the
+ # smaller border-probe radius), so a gapped seam in the
+ # (border, nearest] band can actually be reassigned rather than dropped.
+ for j in tree.query(part.buffer(_ELIM_NEAREST_MAX_DEG)):
+ if j == i:
+ continue
+ other_pc, other_geom, _ = recs[j]
+ if other_pc == pc:
+ continue
+ try:
+ score = part.buffer(_ELIM_SHARED_EPS_DEG).intersection(other_geom).area
+ except GEOSException:
+ score = 0.0
+ # Ties broken by the lexicographically smaller postcode so the
+ # result is independent of STRtree traversal order (matches the
+ # stable tie-break in _resolve_overlaps; keeps output byte-stable
+ # across shapely/GEOS upgrades for content-hash caching).
+ if score > best_score or (
+ score == best_score and best_pc is not None and other_pc < best_pc
+ ):
+ best_score = score
+ best_pc = other_pc
+ dist = part.distance(other_geom)
+ if dist < nearest_d or (
+ dist == nearest_d
+ and nearest_pc is not None
+ and other_pc < nearest_pc
+ ):
+ nearest_d = dist
+ nearest_pc = other_pc
+
+ if best_pc is not None and best_score > 0:
+ assign[i] = best_pc # shares a border -> absorb into that neighbour
+ elif nearest_pc is not None and nearest_d <= _ELIM_NEAREST_MAX_DEG:
+ assign[i] = nearest_pc # gapped seam -> nearest neighbour
+ elif area < _ELIM_DROP_BELOW_M2:
+ assign[i] = None # neighbour-less snap/overlap sliver -> drop
+ # else: keep with its own postcode (a genuine isolated patch)
+
+ if not assign:
+ break
+
+ new_parts: dict[str, list] = defaultdict(list)
+ for i, (pc, part, _) in enumerate(recs):
+ if i in assign:
+ target = assign[i]
+ if target is None:
+ continue
+ new_parts[target].append(part)
+ else:
+ new_parts[pc].append(part)
+
+ rebuilt: dict[str, Polygon | MultiPolygon] = {}
+ for pc, parts in new_parts.items():
+ if len(parts) == 1:
+ rebuilt[pc] = parts[0]
+ else:
+ merged = safe_union(parts, grid=_OUTPUT_PRECISION_DEG)
+ if not merged.is_empty:
+ rebuilt[pc] = merged
+ # Carry forward any postcode that contributed no parts to recs (e.g. an
+ # all-empty input geometry, which the part loop skips): never drop a
+ # postcode just because reassignment fired elsewhere in this pass.
+ for pc, geom in geoms.items():
+ if pc not in rebuilt:
+ rebuilt[pc] = geom
+ geoms = rebuilt
+
+ return list(geoms.items())
+
+
def _round_coords(coords, ndigits=6):
if coords and isinstance(coords[0], (int, float)):
return [round(coords[0], ndigits), round(coords[1], ndigits)]
@@ -501,6 +660,11 @@ def write_district_geojson(
# Remove overlap strips so the output is a clean partition.
converted = _resolve_overlaps(converted)
+ # De-fragment: absorb small detached parts (Voronoi/INSPIRE/seam scatter) into
+ # the neighbour they border most, so postcodes stop shipping as many
+ # disconnected specks. Coverage-preserving; runs on the final partition.
+ converted = _eliminate_small_detached_parts(converted)
+
by_district: dict[str, list[tuple[str, Polygon | MultiPolygon]]] = defaultdict(list)
for pc, geom in converted:
parts = pc.split()
diff --git a/pipeline/transform/postcode_boundaries/test_postcode_boundaries.py b/pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
index c316abb..7f43c78 100644
--- a/pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
+++ b/pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
@@ -4,6 +4,7 @@ Each test targets a specific bug or edge case identified during code review.
"""
import json
+import math
import numpy as np
import polars as pl
@@ -21,6 +22,8 @@ from .inspire import build_inspire_index
from .oa_boundaries import parse_gpkg_geometry
from .greenspace import subtract_greenspace
from .output import (
+ _approx_area_m2,
+ _eliminate_small_detached_parts,
_fill_holes,
merge_fragments,
to_wgs84_geojson,
@@ -1830,3 +1833,231 @@ class TestFragmentsCache:
cache.write_text("c")
# arcgis is optional/absent — it cannot have invalidated the cache.
assert fragments_cache_is_fresh(cache, [tmp_path / "absent.parquet"]) is True
+
+
+# ---------------------------------------------------------------------------
+# De-fragmentation: small detached parts are absorbed into their best neighbour
+# ---------------------------------------------------------------------------
+
+
+# WGS84 helper: build a box from METRE offsets around a fixed England anchor, so
+# the eliminate pass (which works in WGS84 degrees and measures area with a
+# latitude-scaled approximation) sees realistic coordinates and areas.
+_ANCHOR_LON, _ANCHOR_LAT = -0.1, 51.5
+_M_PER_DEG_LAT = 111_320.0
+_M_PER_DEG_LON = 111_320.0 * math.cos(math.radians(_ANCHOR_LAT))
+
+
+def _mbox(x0, y0, x1, y1):
+ return box(
+ _ANCHOR_LON + x0 / _M_PER_DEG_LON,
+ _ANCHOR_LAT + y0 / _M_PER_DEG_LAT,
+ _ANCHOR_LON + x1 / _M_PER_DEG_LON,
+ _ANCHOR_LAT + y1 / _M_PER_DEG_LAT,
+ )
+
+
+def _as_dict(items):
+ return dict(items)
+
+
+class TestEliminateSmallDetachedParts:
+ """A small detached part should be absorbed into the postcode it borders most,
+ de-fragmenting the unit while conserving coverage and never dropping a
+ postcode. Genuine large splits must survive."""
+
+ def test_small_island_absorbed_by_surrounding_neighbour(self):
+ # A: a big main blob (left) plus a small 30x30m island sitting INSIDE B's
+ # territory; B fills the middle/right with a hole where the island is.
+ main_a = _mbox(0, 0, 100, 200) # 20000 m²
+ island = _mbox(150, 90, 180, 120) # 900 m², surrounded by B
+ a = MultiPolygon([main_a, island])
+ b = _mbox(100, 0, 300, 200).difference(island) # hole around the island
+ before = unary_union([a, b]).area
+
+ out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
+
+ assert "A" in out and "B" in out, "no postcode may be dropped"
+ assert out["A"].geom_type == "Polygon", "A's island should be absorbed away"
+ # The island area moves to B (the surrounding postcode); coverage is kept.
+ assert out["B"].contains(island.representative_point())
+ after = unary_union([out["A"], out["B"]]).area
+ assert after == pytest.approx(before, rel=1e-6), "coverage must be conserved"
+
+ def test_genuine_large_split_is_kept(self):
+ # Two substantial parts (both 40000 m²) far apart — a real bisection, not
+ # scatter. Neither is below the absolute/fraction thresholds, so both stay.
+ a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(400, 0, 600, 200)])
+ out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
+ assert out["A"].geom_type == "MultiPolygon"
+ assert len(out["A"].geoms) == 2
+
+ def test_midsize_minor_part_kept_when_above_abs_threshold(self):
+ # A big main plus a 10000 m² detached part: above the 3000 m² absolute
+ # threshold, so it is a genuine secondary piece and must NOT be eliminated,
+ # even though it is a small fraction of the postcode.
+ a = MultiPolygon([_mbox(0, 0, 300, 300), _mbox(1000, 0, 1100, 100)])
+ out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
+ assert out["A"].geom_type == "MultiPolygon"
+
+ def test_tiny_neighbourless_sliver_is_dropped(self):
+ # A main blob plus a 10x10m (100 m²) sliver far from anything: below the
+ # drop threshold with no neighbour to absorb it -> dropped as snap debris.
+ a = MultiPolygon([_mbox(0, 0, 200, 200), _mbox(1000, 1000, 1010, 1010)])
+ out = _as_dict(_eliminate_small_detached_parts([("A", a)]))
+ assert "A" in out, "the postcode itself must survive"
+ assert out["A"].geom_type == "Polygon"
+
+ def test_largest_part_always_retained_no_postcode_dropped(self):
+ # Even a postcode that is ENTIRELY a tiny sliver keeps its (largest) part —
+ # active postcodes must never be dropped (validate_outputs is zero-tolerance).
+ tiny = _mbox(500, 500, 505, 505) # 25 m², the whole postcode
+ big = _mbox(0, 0, 300, 300)
+ out = _as_dict(
+ _eliminate_small_detached_parts([("TINY", tiny), ("BIG", big)])
+ )
+ assert "TINY" in out and not out["TINY"].is_empty
+ assert "BIG" in out
+
+ def test_no_overlaps_introduced(self):
+ # Absorbing parts must keep the output a partition (no double-coverage).
+ main_a = _mbox(0, 0, 100, 200)
+ island = _mbox(150, 90, 180, 120)
+ a = MultiPolygon([main_a, island])
+ b = _mbox(100, 0, 300, 200).difference(island)
+ out = _as_dict(_eliminate_small_detached_parts([("A", a), ("B", b)]))
+ overlap = out["A"].intersection(out["B"]).area
+ assert overlap < (1.0 / (_M_PER_DEG_LAT * _M_PER_DEG_LON)), "no overlap"
+
+ def test_empty_postcode_not_dropped_when_reassignment_fires(self):
+ # Regression: an empty-geometry postcode must survive even when a
+ # reassignment fires elsewhere in the same pass (the rebuild used to drop
+ # any key absent from `recs`). No active postcode may ever be dropped.
+ main_a = _mbox(0, 0, 100, 200)
+ island = _mbox(150, 90, 180, 120) # B absorbs this -> reassignment fires
+ a = MultiPolygon([main_a, island])
+ b = _mbox(100, 0, 300, 200).difference(island)
+ empty = Polygon() # degenerate input that contributes no parts
+ out = _as_dict(
+ _eliminate_small_detached_parts([("A", a), ("B", b), ("EMPTY", empty)])
+ )
+ assert "EMPTY" in out, "an empty-geometry postcode must not be dropped"
+ assert "A" in out and "B" in out
+
+ def test_tiebreak_is_deterministic_smaller_postcode_wins(self):
+ # An island bordered by TWO postcodes with EQUAL shared edges must go to
+ # the lexicographically smaller postcode, independent of STRtree order.
+ a_main = _mbox(0, 0, 100, 100) # A's main, far from the island
+ island = _mbox(300, 50, 320, 70) # 400 m², A's detached part
+ a = MultiPolygon([a_main, island])
+ bbb = _mbox(250, 0, 300, 120) # borders island's left edge
+ ccc = _mbox(320, 0, 370, 120) # borders island's right edge (equal)
+ out = _as_dict(
+ _eliminate_small_detached_parts([("A", a), ("BBB", bbb), ("CCC", ccc)])
+ )
+ pt = island.representative_point()
+ assert out["BBB"].contains(pt), "tie must go to the smaller postcode string"
+ assert not out["CCC"].contains(pt)
+ assert out["A"].geom_type == "Polygon"
+
+ def test_gapped_sliver_within_nearest_radius_is_reassigned_not_dropped(self):
+ # A 300 m² sliver 1.2 m from its only neighbour N — beyond the border probe
+ # but inside the nearest-fallback radius. Before the gather buffer was
+ # widened to the accept radius, the old (smaller) gather buffer never
+ # returned N, so the sliver fell through to the drop branch. Now it is
+ # reassigned to N, conserving coverage.
+ x_main = _mbox(0, 0, 100, 100) # far from the sliver
+ x_sliver = _mbox(278.8, 50, 298.8, 65) # 300 m², right edge at x=298.8
+ x = MultiPolygon([x_main, x_sliver])
+ n = _mbox(300, 0, 400, 200) # left edge at x=300 -> 1.2 m gap
+ before = unary_union([x, n]).area
+ out = _as_dict(_eliminate_small_detached_parts([("X", x), ("N", n)]))
+ assert out["X"].geom_type == "Polygon", "sliver should leave X"
+ assert out["N"].contains(x_sliver.representative_point()), "absorbed into N"
+ after = unary_union([out["X"], out["N"]]).area
+ assert after == pytest.approx(before, rel=1e-6), "coverage conserved, not dropped"
+
+ def test_approx_area_matches_real_metric_area(self):
+ # The latitude-scaled area approximation should be within a few percent of
+ # the true projected area for a building-scale polygon.
+ import pyproj
+ from shapely.ops import transform as transform_geometry
+
+ poly = _mbox(0, 0, 50, 60) # ~3000 m²
+ to_bng = pyproj.Transformer.from_crs(
+ "EPSG:4326", "EPSG:27700", always_xy=True
+ )
+ true_m2 = transform_geometry(to_bng.transform, poly).area
+ assert _approx_area_m2(poly) == pytest.approx(true_m2, rel=0.02)
+
+
+# ---------------------------------------------------------------------------
+# Greenspace subtraction is connectivity-preserving
+# ---------------------------------------------------------------------------
+
+
+class TestGreenspaceReconnect:
+ """Greenspace that CROSSES a postcode must not shatter it: a narrow strip is
+ bridged back (postcode stays one piece), a wide barrier is left subtracted
+ (postcode genuinely splits)."""
+
+ def test_narrow_strip_reconnects_postcode(self):
+ from shapely.strtree import STRtree
+
+ postcode = box(0, 0, 200, 100)
+ strip = box(95, 0, 105, 100) # 10 m wide green strip crossing the postcode
+ # Sanity: a plain difference WOULD split it into two parts.
+ assert postcode.difference(strip).geom_type == "MultiPolygon"
+
+ result = subtract_greenspace(postcode, STRtree([strip]), [strip])
+ assert result.geom_type == "Polygon", "narrow strip should be bridged"
+ assert result.is_valid
+
+ def test_wide_barrier_keeps_postcode_split(self):
+ from shapely.strtree import STRtree
+
+ postcode = box(0, 0, 200, 100)
+ barrier = box(60, 0, 130, 100) # 70 m wide — beyond 2x the bridge width
+ result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
+ assert result.geom_type == "MultiPolygon", "wide barrier should stay split"
+ assert len(result.geoms) == 2
+
+ def test_edge_greenspace_still_trimmed(self):
+ # Greenspace on the edge (not crossing) is trimmed as before; reconnect is
+ # a no-op because the result stays a single piece.
+ from shapely.strtree import STRtree
+
+ postcode = box(0, 0, 100, 100) # 10000 m²
+ park = box(60, 0, 100, 100) # 4000 m² on the right edge
+ result = subtract_greenspace(postcode, STRtree([park]), [park])
+ assert result.geom_type == "Polygon"
+ assert result.area == pytest.approx(6000, rel=0.01)
+
+ def test_result_is_always_polygonal(self):
+ # Regression: _reconnect_split must never return a GeometryCollection
+ # (line/point debris) — downstream to_wgs84_geojson_multi would truncate a
+ # GC to a single part, silently dropping substantial pieces. Sweep many
+ # strip widths/offsets (incl. coincident-edge-prone integer geometries).
+ from shapely.strtree import STRtree
+
+ postcode = box(0, 0, 200, 120)
+ for w in (2, 5, 8, 10, 25, 49, 50, 51, 60, 90):
+ for x0 in (40, 70, 95, 100, 130):
+ strip = box(x0, -10, x0 + w, 130)
+ result = subtract_greenspace(postcode, STRtree([strip]), [strip])
+ assert result.geom_type in ("Polygon", "MultiPolygon"), (
+ f"w={w} x0={x0} produced {result.geom_type}"
+ )
+ assert result.is_valid and not result.is_empty
+
+ def test_wide_barrier_preserves_all_substantial_parts(self):
+ # The motivating case for the GC fix: a wide barrier genuinely splits the
+ # postcode; ALL substantial parts must survive (not be truncated to one).
+ from shapely.strtree import STRtree
+
+ postcode = box(0, 0, 300, 100)
+ barrier = box(120, 0, 200, 100) # 80 m wide -> beyond 2x bridge
+ result = subtract_greenspace(postcode, STRtree([barrier]), [barrier])
+ assert result.geom_type == "MultiPolygon"
+ areas = sorted(p.area for p in result.geoms)
+ assert areas == pytest.approx([10000, 12000], rel=0.01) # both banks kept
diff --git a/pipeline/transform/price_estimation/estimate.py b/pipeline/transform/price_estimation/estimate.py
index efc51c9..7147cd2 100644
--- a/pipeline/transform/price_estimation/estimate.py
+++ b/pipeline/transform/price_estimation/estimate.py
@@ -1,4 +1,4 @@
-"""Augment properties.parquet with estimated current prices.
+"""Estimate current prices for the merged properties, as a standalone artifact.
For properties with a known prior sale, applies the repeat-sales price index
to adjust the last known price to the current date, then blends with kNN
@@ -6,8 +6,13 @@ estimates from nearby recently-sold properties. Includes:
- Capping extreme index adjustments
- kNN spatial blending
-Modifies properties.parquet in-place. Temporarily joins postcode.parquet
-for lat/lon needed by kNN, then drops those columns before writing.
+Reads the slim price_inputs.parquet (built by property_base, independently of
+merge's area features) plus postcode.parquet for the lat/lon kNN needs, and
+writes a slim price_estimates.parquet of just the natural key (Postcode +
+coalesced address) and "Estimated current price" / "Est. price per sqm".
+join_price_estimates.py joins those two columns back onto properties.parquet.
+Because the inputs do not depend on merge's area columns, adding such a column
+does not invalidate this step.
"""
import argparse
@@ -26,12 +31,27 @@ from pipeline.transform.price_estimation.knn import (
from pipeline.transform.price_estimation.utils import (
CURRENT_FRAC_YEAR,
CURRENT_YEAR,
+ ESTIMATE_COLUMNS,
+ JOIN_KEYS,
MAX_LOG_ADJUSTMENT,
interpolate_log_index,
+ join_address_expr,
sector_expr,
type_group_expr,
)
+# Columns estimate reads from price_inputs.parquet. The two address columns are
+# only carried to build the natural join key (Postcode + coalesced address).
+INPUT_COLUMNS = [
+ "Postcode",
+ "Property type",
+ "Total floor area (sqm)",
+ "Last known price",
+ "Date of last transaction",
+ "Address per Property Register",
+ "Address per EPC",
+]
+
MAX_KNN_TO_INDEX_RATIO = 2.0
MIN_KNN_TO_INDEX_RATIO = 0.5
# Cap the final estimate at this multiple of the last known price as a guard
@@ -161,13 +181,14 @@ def guarded_blend_estimates(
def main():
parser = argparse.ArgumentParser(
- description="Augment properties.parquet with estimated current prices"
+ description="Estimate current prices for the merged properties"
)
parser.add_argument(
- "--properties",
+ "--input",
type=Path,
required=True,
- help="Path to properties.parquet (modified in-place)",
+ help="Path to price_inputs.parquet (slim per-dwelling inputs from "
+ "property_base)",
)
parser.add_argument(
"--postcodes",
@@ -178,22 +199,23 @@ def main():
parser.add_argument(
"--index", type=Path, required=True, help="Path to price_index.parquet"
)
+ parser.add_argument(
+ "--output",
+ type=Path,
+ required=True,
+ help="Output price_estimates.parquet (natural key + estimate columns)",
+ )
args = parser.parse_args()
- print("Loading properties.parquet...")
- df = pl.read_parquet(args.properties)
- print(f" {len(df):,} rows, {len(df.columns)} columns")
+ print("Loading price inputs (projection)...")
+ df = pl.read_parquet(args.input, columns=INPUT_COLUMNS)
+ print(f" {len(df):,} rows, {len(INPUT_COLUMNS)} input columns")
# Join lat/lon from postcode.parquet for kNN spatial queries
postcodes = pl.read_parquet(args.postcodes).select("Postcode", "lat", "lon")
df = df.join(postcodes, on="Postcode", how="left")
print(f" Joined lat/lon from {len(postcodes):,} postcodes")
- # Drop existing estimated columns if re-running
- for col in ["Estimated current price", "Est. price per sqm"]:
- if col in df.columns:
- df = df.drop(col)
-
# Derive helper columns
df = df.with_columns(
sector_expr().alias("_sector"),
@@ -355,16 +377,15 @@ def main():
.alias("Est. price per sqm"),
)
- # Drop all temporary columns and joined lat/lon (those belong in postcode.parquet)
- temp_cols = [c for c in df.columns if c.startswith("_") or c.startswith("log_idx_")]
- df = df.drop(temp_cols).drop("lat", "lon")
+ # Emit only the natural join key and the two estimate columns.
+ # join_price_estimates.py joins these back onto properties.parquet.
+ result = df.with_columns(join_address_expr()).select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
- df.write_parquet(args.properties)
- size_mb = args.properties.stat().st_size / (1024 * 1024)
- print(f"\nWrote {args.properties} ({size_mb:.1f} MB)")
- print(
- f" {len(df):,} rows, {len(df.columns)} columns (including 'Estimated current price')"
- )
+ result.write_parquet(args.output)
+ size_mb = args.output.stat().st_size / (1024 * 1024)
+ print(f"\nWrote {args.output} ({size_mb:.1f} MB)")
+ n_priced = result.filter(pl.col("Estimated current price").is_not_null()).height
+ print(f" {len(result):,} rows, {n_priced:,} with an estimated current price")
if __name__ == "__main__":
diff --git a/pipeline/transform/price_estimation/utils.py b/pipeline/transform/price_estimation/utils.py
index 2b53346..e551e17 100644
--- a/pipeline/transform/price_estimation/utils.py
+++ b/pipeline/transform/price_estimation/utils.py
@@ -16,6 +16,26 @@ LATEST_COMPLETE_YEAR = CURRENT_YEAR - 1
_today = date.today()
CURRENT_FRAC_YEAR = _today.year + (_today.month - 1) / 12
+# The two columns price estimation contributes to properties.parquet, kept here
+# so both the producer (estimate) and the joiner (join_price_estimates) agree.
+ESTIMATE_COLUMNS = ["Estimated current price", "Est. price per sqm"]
+
+# Natural join key from estimates back onto properties: postcode plus the
+# coalesced register/EPC address. This is unique and non-null on the deduped
+# dwelling universe (see property_base._dedupe_collapsed_properties), so it maps
+# estimates 1:1 onto properties regardless of row order — estimates are computed
+# from a separate price_inputs.parquet, so a positional key would not line up.
+JOIN_ADDRESS = "_join_address"
+JOIN_KEYS = ["Postcode", JOIN_ADDRESS]
+
+
+def join_address_expr() -> pl.Expr:
+ """The coalesced address half of the natural key, aliased to JOIN_ADDRESS."""
+ return pl.coalesce("Address per Property Register", "Address per EPC").alias(
+ JOIN_ADDRESS
+ )
+
+
# Cap on log(index_ratio) to prevent wild estimates from thin sectors
MAX_LOG_ADJUSTMENT = 3.0 # ~20x max price change
TERRACE_TYPES = [
diff --git a/pipeline/transform/property_base.py b/pipeline/transform/property_base.py
new file mode 100644
index 0000000..cde7c3a
--- /dev/null
+++ b/pipeline/transform/property_base.py
@@ -0,0 +1,217 @@
+"""Shared property base: the dwelling universe before any area enrichment.
+
+This is the single source of truth for *which* dwellings exist and their
+intrinsic, source-level attributes (price, floor area, type, addresses). Both
+``merge`` (which enriches it with postcode/LSOA-keyed area features to build
+properties.parquet) and price estimation (which only needs the intrinsic
+columns) start from exactly these rows, so estimates computed here line up 1:1
+with the final properties by the natural key ``(Postcode, coalesced address)``.
+
+Living in its own module is what lets price estimation be *cached* across
+merge changes: ``price_inputs.parquet`` depends only on epc_pp + arcgis + this
+file, so adding an area column to merge.py does not invalidate it and the
+expensive index/kNN steps are skipped.
+"""
+
+import argparse
+from pathlib import Path
+
+import polars as pl
+
+from pipeline.utils.postcode_mapping import build_postcode_mapping
+
+MIN_FLOOR_AREA_M2 = 10
+
+# Columns price estimation reads, with the final (properties.parquet) names so
+# index.py/estimate.py and the join all speak the same schema. The two address
+# columns form the natural join key (Postcode + their coalesce).
+PRICE_INPUT_SELECT = [
+ pl.col("postcode").alias("Postcode"),
+ pl.col("total_floor_area").alias("Total floor area (sqm)"),
+ pl.col("latest_price").alias("Last known price"),
+ pl.col("date_of_transfer").alias("Date of last transaction"),
+ "historical_prices",
+ pl.col("pp_address").alias("Address per Property Register"),
+ pl.col("epc_address").alias("Address per EPC"),
+]
+
+
+def _active_english_postcode_area(arcgis_raw: pl.LazyFrame) -> pl.LazyFrame:
+ """Return the supported postcode universe with geography join keys."""
+ return (
+ arcgis_raw.filter(pl.col("ctry25cd") == "E92000001")
+ .filter(pl.col("doterm").is_null())
+ .select(
+ pl.col("pcds").alias("postcode"),
+ "lat",
+ pl.col("long").alias("lon"),
+ "ctry25cd",
+ pl.col("lsoa21cd").alias("lsoa21"),
+ pl.col("oa21cd").alias("oa21"),
+ pl.col("pcon24cd").alias("pcon"),
+ )
+ .drop_nulls(["postcode"])
+ .unique(["postcode"])
+ )
+
+
+def _remap_terminated_postcodes(
+ wide: pl.LazyFrame, postcode_mapping: pl.LazyFrame
+) -> pl.LazyFrame:
+ return (
+ wide.join(
+ postcode_mapping,
+ left_on="postcode",
+ right_on="old_postcode",
+ how="left",
+ )
+ .with_columns(
+ pl.coalesce("new_postcode", "postcode").alias("postcode"),
+ )
+ .drop("new_postcode")
+ )
+
+
+def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
+ """Keep one row per (postcode, address) — the most-recent transaction.
+
+ The terminated-postcode remap can map two distinct postcodes onto one active
+ successor, collapsing the same physical address onto a single
+ (postcode, address) key with conflicting sale records. Keep the row with the
+ latest date_of_transfer so the headline price/date reflect the most recent
+ transaction; genuinely distinct addresses are untouched.
+
+ The dedup key coalesces the price-paid address with the EPC address: EPC-only
+ dwellings (never sold) have a null pp_address, so keying on pp_address alone
+ would collapse EVERY EPC-only dwelling in a postcode onto one
+ (postcode, null) key and silently drop all but one. Each dwelling's coalesced
+ address is unique within its postcode (the EPC frame is deduped on
+ address+postcode upstream), so the coalesced key keeps them distinct while
+ leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
+ a sale exists.
+ """
+ return (
+ wide.with_columns(
+ pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
+ )
+ .sort("date_of_transfer", descending=True, nulls_last=True)
+ .unique(
+ subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
+ )
+ .drop("_dedupe_address")
+ )
+
+
+def _filter_to_active_english_postcodes(
+ wide: pl.LazyFrame, active_postcodes: pl.LazyFrame
+) -> pl.LazyFrame:
+ return wide.join(active_postcodes, on="postcode", how="semi")
+
+
+def property_type_expr() -> pl.Expr:
+ """Unaliased property-type expression: prefer EPC, fall back to price-paid.
+
+ For Houses, use built_form (e.g. Semi-Detached, Mid-Terrace) for finer
+ detail. Depends only on intrinsic base columns (epc_property_type,
+ pp_property_type, built_form), so merge and price_inputs derive the same
+ value. Callers alias it ("property_type" in merge, "Property type" in
+ price_inputs).
+ """
+ bad_built_form = pl.col("built_form").is_null() | pl.col("built_form").is_in(
+ ["NO DATA!", "Not Recorded"]
+ )
+ has_epc = pl.col("epc_property_type").is_not_null()
+ is_house = pl.col("epc_property_type") == "House"
+ return (
+ pl.when(has_epc & is_house & ~bad_built_form)
+ .then(pl.col("built_form"))
+ .when(has_epc & is_house)
+ .then(pl.col("pp_property_type"))
+ .when(has_epc)
+ .then(pl.col("epc_property_type"))
+ .otherwise(pl.col("pp_property_type"))
+ # Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
+ # collapse terrace sub-types, and fold rare types into "Other"
+ .replace(
+ {
+ "Flat": "Flats/Maisonettes",
+ "Maisonette": "Flats/Maisonettes",
+ "End-Terrace": "Terraced",
+ "Mid-Terrace": "Terraced",
+ "Enclosed End-Terrace": "Terraced",
+ "Enclosed Mid-Terrace": "Terraced",
+ "Bungalow": "Other",
+ "Park home": "Other",
+ }
+ )
+ )
+
+
+def build_postcode_centroids(arcgis_path: Path) -> pl.LazyFrame:
+ """One row per active-English postcode with its lat/lon, from arcgis.
+
+ This is the lat/lon source price estimation needs (index sector centroids,
+ kNN). It is the same per-postcode lat/lon merge writes into postcode.parquet
+ (both come from arcgis), but built straight from arcgis so the index/estimate
+ steps do not depend on the merge output — adding an area column to merge
+ therefore does not invalidate the expensive price index/kNN.
+ """
+ return _active_english_postcode_area(pl.scan_parquet(arcgis_path)).select(
+ pl.col("postcode").alias("Postcode"), "lat", "lon"
+ )
+
+
+def build_property_base(epc_pp_path: Path, arcgis_path: Path) -> pl.LazyFrame:
+ """The deduped, active-English dwelling universe from epc_pp + arcgis.
+
+ Floor filter -> terminated-postcode remap -> collapse-dedupe -> restrict to
+ the active English postcode universe. Returns a LazyFrame with the original
+ epc_pp column names; merge enriches it, the CLI projects it to price_inputs.
+ """
+ wide = pl.scan_parquet(epc_pp_path).filter(
+ pl.col("total_floor_area").is_null()
+ | (pl.col("total_floor_area") > MIN_FLOOR_AREA_M2)
+ )
+ postcode_mapping = build_postcode_mapping(arcgis_path)
+ wide = _remap_terminated_postcodes(wide, postcode_mapping.lazy())
+ wide = _dedupe_collapsed_properties(wide)
+ active_postcodes = (
+ _active_english_postcode_area(pl.scan_parquet(arcgis_path))
+ .select("postcode")
+ .unique()
+ )
+ return _filter_to_active_english_postcodes(wide, active_postcodes)
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Write the slim price-estimation inputs from epc_pp + arcgis"
+ )
+ parser.add_argument("--epc-pp", type=Path, required=True)
+ parser.add_argument("--arcgis", type=Path, required=True)
+ parser.add_argument(
+ "--output", type=Path, required=True, help="price_inputs.parquet output"
+ )
+ parser.add_argument(
+ "--centroids",
+ type=Path,
+ required=True,
+ help="postcode_centroids.parquet output (Postcode, lat, lon)",
+ )
+ args = parser.parse_args()
+
+ base = build_property_base(args.epc_pp, args.arcgis)
+ price_inputs = base.with_columns(
+ property_type_expr().alias("Property type")
+ ).select(*PRICE_INPUT_SELECT, "Property type")
+ price_inputs.sink_parquet(args.output)
+ n = pl.scan_parquet(args.output).select(pl.len()).collect().item()
+ print(f"Wrote {args.output} ({args.output.stat().st_size / 1e6:.1f} MB), {n:,} dwellings")
+
+ build_postcode_centroids(args.arcgis).sink_parquet(args.centroids)
+ n_pc = pl.scan_parquet(args.centroids).select(pl.len()).collect().item()
+ print(f"Wrote {args.centroids} ({args.centroids.stat().st_size / 1e6:.1f} MB), {n_pc:,} postcodes")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pipeline/transform/test_area_crime_averages.py b/pipeline/transform/test_area_crime_averages.py
new file mode 100644
index 0000000..5062d30
--- /dev/null
+++ b/pipeline/transform/test_area_crime_averages.py
@@ -0,0 +1,81 @@
+import polars as pl
+
+from pipeline.transform.area_crime_averages import (
+ NATIONAL_AREA,
+ SCOPE_NATIONAL,
+ SCOPE_OUTCODE,
+ SCOPE_SECTOR,
+ compute_area_crime_averages,
+)
+
+_BURGLARY = "Burglary (/yr, 7y)"
+_ROBBERY = "Robbery (/yr, 7y)"
+
+
+def _postcodes() -> pl.LazyFrame:
+ return pl.LazyFrame(
+ {
+ "Postcode": ["E14 2DG", "E14 2AB", "E14 9XY", "M1 1AE", "E14 2ZZ"],
+ # E14 9XY has no usable crime data; E14 2AB lacks robbery; E14 2ZZ has
+ # crime but (below) no properties, so it must not weight any average.
+ _BURGLARY: [10.0, 20.0, None, 5.0, 100.0],
+ _ROBBERY: [2.0, None, None, 1.0, 50.0],
+ # An unrelated column proves only the crime columns are averaged.
+ "Median age": [40.0, 41.0, 42.0, 30.0, 99.0],
+ }
+ )
+
+
+def _properties() -> pl.LazyFrame:
+ # Property rows per postcode become the weights (3 / 1 / 2 / 4). E14 2ZZ has
+ # none, so it is excluded entirely.
+ postcodes = ["E14 2DG"] * 3 + ["E14 2AB"] + ["E14 9XY"] * 2 + ["M1 1AE"] * 4
+ return pl.LazyFrame({"Postcode": postcodes})
+
+
+def _row(df: pl.DataFrame, scope: str, area: str) -> dict:
+ matched = df.filter((pl.col("scope") == scope) & (pl.col("area") == area))
+ assert matched.height == 1, f"expected one {scope} row for {area!r}"
+ return matched.to_dicts()[0]
+
+
+def test_property_weighted_means_skip_nulls():
+ result = compute_area_crime_averages(_postcodes(), _properties())
+
+ national = _row(result, SCOPE_NATIONAL, NATIONAL_AREA)
+ # Burglary: (10*3 + 20*1 + 5*4) / (3+1+4) = 70/8; E14 9XY null dilutes nothing.
+ assert national[_BURGLARY] == 8.75
+ # Robbery: (2*3 + 1*4) / (3+4) = 10/7; both null postcodes are excluded from
+ # the numerator AND the denominator.
+ assert national[_ROBBERY] == pl.Series([10.0 / 7.0]).cast(pl.Float32).item()
+
+ outcode = _row(result, SCOPE_OUTCODE, "E14")
+ assert outcode[_BURGLARY] == 12.5 # (10*3 + 20*1) / 4
+ assert outcode[_ROBBERY] == 2.0 # only E14 2DG has robbery (2 * 3 / 3)
+
+
+def test_sector_aggregation_and_all_null_rows_dropped():
+ result = compute_area_crime_averages(_postcodes(), _properties())
+
+ sector = _row(result, SCOPE_SECTOR, "E14 2")
+ assert sector[_BURGLARY] == 12.5
+ assert sector[_ROBBERY] == 2.0
+
+ # E14 9XY has properties but no crime data at all, so its sector "E14 9" is
+ # all-null and must be dropped rather than reported as a known area.
+ assert result.filter(pl.col("area") == "E14 9").height == 0
+
+
+def test_postcodes_without_properties_are_excluded():
+ result = compute_area_crime_averages(_postcodes(), _properties())
+
+ # E14 2ZZ carries crime values but no properties; including it would pull the
+ # E14 outcode burglary mean toward its 100.0. It must contribute nothing.
+ outcode = _row(result, SCOPE_OUTCODE, "E14")
+ assert outcode[_BURGLARY] == 12.5
+
+
+def test_only_crime_columns_are_emitted():
+ result = compute_area_crime_averages(_postcodes(), _properties())
+ assert set(result.columns) == {"scope", "area", _BURGLARY, _ROBBERY}
+ assert result.schema[_BURGLARY] == pl.Float32
diff --git a/pipeline/transform/test_crime_spatial.py b/pipeline/transform/test_crime_spatial.py
index 22d6f6c..39dacd4 100644
--- a/pipeline/transform/test_crime_spatial.py
+++ b/pipeline/transform/test_crime_spatial.py
@@ -1,13 +1,10 @@
import json
-import numpy as np
import polars as pl
import pytest
-import shapely
from pyproj import Transformer
from pipeline.transform.crime_spatial import transform_crime_spatial
-from pipeline.transform.postcode_boundaries.loader import load_postcode_polygons
_TO_WGS84 = Transformer.from_crs("EPSG:27700", "EPSG:4326", always_xy=True)
@@ -16,6 +13,10 @@ _CSV_HEADER = (
"LSOA code,LSOA name,Crime type,Last outcome category,Context"
)
+# Average-annual-count crime column name for a window (the filterable feature).
+def _raw(t: str, window: str = "7y") -> str:
+ return f"{t} (/yr, {window})"
+
def _bng_to_wgs84(x: float, y: float) -> tuple[float, float]:
lon, lat = _TO_WGS84.transform(x, y)
@@ -39,12 +40,12 @@ def _write_boundaries(units_dir, features_by_district: dict[str, list[dict]]) ->
(units_dir / f"{district}.geojson").write_text(json.dumps(collection))
-def _crime_row(month: str, x, y, crime_type: str) -> str:
+def _crime_row(month: str, x, y, crime_type: str, location="On or near X", outcome="U") -> str:
if x is None or y is None:
lon, lat = "", ""
else:
lon, lat = _bng_to_wgs84(x, y)
- return f",{month},F,F,{lon},{lat},On or near X,E01000001,L,{crime_type},U,"
+ return f",{month},F,F,{lon},{lat},{location},E01000001,L,{crime_type},{outcome},"
def _write_month(
@@ -59,10 +60,22 @@ def _write_month(
def _run(tmp_path, crime, units, **kwargs):
- output = tmp_path / "crime_by_postcode.parquet"
+ """Run the transform and return (crime, by_year, records) DataFrames.
+
+ The crime table carries the average-annual-count columns ("{type} (/yr, …)"),
+ i.e. the raw, absolute number of recorded incidents per year.
+ """
+ crime_out = tmp_path / "crime_by_postcode.parquet"
by_year = tmp_path / "crime_by_postcode_by_year.parquet"
- transform_crime_spatial(crime, units, output, by_year, buffer_m=50.0, **kwargs)
- return pl.read_parquet(output), pl.read_parquet(by_year)
+ records = tmp_path / "crime_records.parquet"
+ transform_crime_spatial(
+ crime, units, crime_out, by_year, records, buffer_m=50.0, **kwargs
+ )
+ return (
+ pl.read_parquet(crime_out),
+ pl.read_parquet(by_year),
+ pl.read_parquet(records),
+ )
def test_buffer_overlap_counts_for_each_postcode(tmp_path):
@@ -95,17 +108,74 @@ def test_buffer_overlap_counts_for_each_postcode(tmp_path):
],
)
- avg_df, _ = _run(tmp_path, crime, units)
- rows = {r["postcode"]: r for r in avg_df.to_dicts()}
- # Single covered month -> pooled rate x12.
- assert rows["AB1 1AA"]["Burglary (avg/yr)"] == 12.0
- assert rows["AB1 1AB"]["Burglary (avg/yr)"] == 12.0
- assert rows["AB1 1AA"]["Robbery (avg/yr)"] == 0.0
+ raw_df, _, _ = _run(tmp_path, crime, units)
+ rows = {r["postcode"]: r for r in raw_df.to_dicts()}
+ # Single covered month -> pooled raw rate x12.
+ assert rows["AB1 1AA"][_raw("Burglary")] == 12.0
+ assert rows["AB1 1AB"][_raw("Burglary")] == 12.0
+ assert rows["AB1 1AA"][_raw("Robbery")] == 0.0
# Only the 49m robbery counts for C; the 51m one and the blank row do not.
- assert rows["AB1 1AC"]["Robbery (avg/yr)"] == 12.0
- assert rows["AB1 1AC"]["Burglary (avg/yr)"] == 0.0
+ assert rows["AB1 1AC"][_raw("Robbery")] == 12.0
+ assert rows["AB1 1AC"][_raw("Burglary")] == 0.0
# Anti-social behaviour had no coordinate -> nobody gets it.
- assert all(r["Anti-social behaviour (avg/yr)"] == 0.0 for r in rows.values())
+ assert all(r[_raw("Anti-social behaviour")] == 0.0 for r in rows.values())
+
+
+def test_counts_are_not_area_normalised(tmp_path):
+ # Three postcodes of very different footprint, each with exactly one incident
+ # in its buffer. The raw count must be 12/yr for ALL of them: area
+ # normalisation has been removed, so footprint no longer changes the number.
+ units = tmp_path / "units"
+ _write_boundaries(
+ units,
+ {
+ "AB1": [
+ _square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
+ _square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20
+ _square_feature("AB1 1AC", 5000, 5000, 5040, 5040), # 40x40
+ ]
+ },
+ )
+
+ crime = tmp_path / "crime"
+ _write_month(
+ crime,
+ "2024-01",
+ [
+ _crime_row("2024-01", 1005, 1005, "Burglary"),
+ _crime_row("2024-01", 3005, 3010, "Burglary"),
+ _crime_row("2024-01", 5020, 5020, "Burglary"),
+ ],
+ )
+
+ raw_df, _, _ = _run(tmp_path, crime, units, min_bar_months=1)
+ rows = {r["postcode"]: r for r in raw_df.to_dicts()}
+ for pc in ("AB1 1AA", "AB1 1AB", "AB1 1AC"):
+ assert rows[pc][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
+
+
+def test_windows_pool_only_recent_years(tmp_path):
+ # 2-year window vs 7-year window. An incident in the latest year sits in both
+ # windows; one 6 years back sits only in the 7-year window.
+ units = tmp_path / "units"
+ _write_boundaries(
+ units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
+ )
+ crime = tmp_path / "crime"
+ # 12 covered months in 2019 (1 burglary), 12 in 2025 (1 burglary). Latest =
+ # 2025: 7y window = 2019..2025 (both), 2y window = 2024..2025 (only 2025).
+ for month in range(1, 13):
+ ym19 = f"2019-{month:02d}"
+ ym25 = f"2025-{month:02d}"
+ _write_month(crime, ym19, [_crime_row(ym19, 1005, 1005, "Burglary")] if month == 1 else [])
+ _write_month(crime, ym25, [_crime_row(ym25, 1005, 1005, "Burglary")] if month == 1 else [])
+
+ raw_df, _, _ = _run(tmp_path, crime, units)
+ row = raw_df.row(0, named=True)
+ # 7y: 2 incidents over 24 covered months -> 1/yr.
+ assert row[_raw("Burglary", "7y")] == pytest.approx(1.0, abs=0.05)
+ # 2y: 1 incident over 12 covered months -> 1/yr (the 2019 one is excluded).
+ assert row[_raw("Burglary", "2y")] == pytest.approx(1.0, abs=0.05)
def test_by_year_annualises_and_rolls_up(tmp_path):
@@ -115,7 +185,6 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
)
crime = tmp_path / "crime"
- # Point at the centre of AB1 1AA, well inside its buffer.
_write_month(
crime,
"2023-01",
@@ -134,7 +203,7 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
],
)
- _, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
+ _, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
assert by_year_df.height == 1
cols = set(by_year_df.columns)
assert {"Burglary (by year)", "Serious crime (by year)", "Minor crime (by year)"} <= cols
@@ -150,77 +219,14 @@ def test_by_year_annualises_and_rolls_up(tmp_path):
# 2023 serious = Burglary(12) + Robbery(12) = 24; 2024 = Burglary(12).
assert serious[2023] == 24.0
assert serious[2024] == 12.0
- # Coverage calendar: both years published, with their month counts.
coverage = {c["year"]: c["months"] for c in row["covered_years"]}
assert coverage == {2023: 1, 2024: 2}
-def test_area_normalisation_divides_out_buffered_catchment(tmp_path):
- # Three postcodes of increasing footprint, each with exactly one incident in
- # its buffer. Normalisation rescales by median_catchment / buffered_area, so
- # the smallest scores highest and the median-sized one is unchanged -- i.e.
- # the metric is a density. Dividing by the *buffered* catchment (not the raw
- # polygon) means the fixed buffer-ring floor keeps the spread gentle, so the
- # tiniest postcode is not blown up out of proportion.
- units = tmp_path / "units"
- _write_boundaries(
- units,
- {
- "AB1": [
- _square_feature("AB1 1AA", 1000, 1000, 1010, 1010), # 10x10
- _square_feature("AB1 1AB", 3000, 3000, 3010, 3020), # 10x20 (median)
- _square_feature("AB1 1AC", 5000, 5000, 5020, 5020), # 20x20
- ]
- },
- )
-
- crime = tmp_path / "crime"
- _write_month(
- crime,
- "2024-01",
- [
- _crime_row("2024-01", 1005, 1005, "Burglary"),
- _crime_row("2024-01", 3005, 3010, "Burglary"),
- _crime_row("2024-01", 5010, 5010, "Burglary"),
- ],
- )
-
- avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
-
- # Re-derive the expected values from the same buffered catchment areas: each
- # postcode is 12/yr before normalisation, then x (median_buf / buffered_area).
- postcodes, polygons = load_postcode_polygons(units)
- buf_area = {
- pc: float(shapely.area(shapely.buffer(poly, 50.0, quad_segs=8)))
- for pc, poly in zip(postcodes, polygons)
- }
- median_buf = float(np.median(list(buf_area.values())))
- expected = {pc: 12.0 * median_buf / buf_area[pc] for pc in buf_area}
-
- rows = {r["postcode"]: r for r in avg_df.to_dicts()}
- for pc, exp in expected.items():
- assert rows[pc]["Burglary (avg/yr)"] == pytest.approx(exp, abs=0.1)
-
- # Median catchment unchanged; ordering is by inverse buffered area, but the
- # buffer-ring floor keeps the spread far below the ~4x raw-area ratio.
- assert rows["AB1 1AB"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
- small = rows["AB1 1AA"]["Burglary (avg/yr)"]
- big = rows["AB1 1AC"]["Burglary (avg/yr)"]
- assert small > 12.0 > big
- assert small / big < 1.5
-
- # by-year series carries the same normalisation.
- small_row = by_year_df.filter(pl.col("postcode") == "AB1 1AA").row(0, named=True)
- assert small_row["Burglary (by year)"] == [
- {"year": 2024, "count": pytest.approx(expected["AB1 1AA"], abs=0.1)}
- ]
-
-
-def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
- # Uneven month coverage across years: 2023 has 1 month (2 incidents),
- # 2024 has 2 months (2 incidents). The headline is the POOLED annualised
- # rate over all covered months: 4 incidents / 3 months * 12 = 16/yr -- not
- # the old mean-of-bars (24+12)/2 = 18, which over-weighted thin years.
+def test_raw_is_pooled_rate_over_covered_months(tmp_path):
+ # Uneven month coverage: 2023 has 1 month (2 incidents), 2024 has 2 months
+ # (2 incidents). The raw figure is the POOLED annualised rate over all covered
+ # months: 4 incidents / 3 months * 12 = 16/yr.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@@ -238,10 +244,9 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
_write_month(crime, "2024-02", [_crime_row("2024-02", 1005, 1005, "Burglary")])
- avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
- avg = avg_df.row(0, named=True)
- assert avg["Burglary (avg/yr)"] == pytest.approx(16.0, abs=0.05)
+ assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(16.0, abs=0.05)
# Bars remain per-year annualised: 2023 -> 24/yr (x12), 2024 -> 12/yr (x6).
row = by_year_df.row(0, named=True)
@@ -251,8 +256,7 @@ def test_avg_yr_is_pooled_rate_over_covered_months(tmp_path):
def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
# A single robbery in a 24-covered-month window must read as ~0.5/yr (the
- # long-run pooled rate), NOT 12/yr (the old years-with-incidents mean that
- # inflated sporadic categories by up to ~15x).
+ # long-run pooled rate), NOT 12/yr.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@@ -266,14 +270,10 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
rows = [_crime_row(f"{year}-{month:02d}", 1005, 1005, "Robbery")]
_write_month(crime, f"{year}-{month:02d}", rows)
- avg_df, by_year_df = _run(tmp_path, crime, units)
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units)
- avg = avg_df.row(0, named=True)
# 1 incident over 24 covered months -> 0.5/yr.
- assert avg["Robbery (avg/yr)"] == pytest.approx(0.5, abs=0.05)
- # The by-year bar still shows the 2023 incident annualised over 12 covered
- # months (1/yr); 2024 is covered with zero robberies -> no bar, but the
- # year IS in the coverage list so consumers may render it as a true zero.
+ assert raw_df.row(0, named=True)[_raw("Robbery")] == pytest.approx(0.5, abs=0.05)
row = by_year_df.row(0, named=True)
bars = {p["year"]: p["count"] for p in row["Robbery (by year)"]}
assert bars == {2023: pytest.approx(1.0, abs=0.05)}
@@ -283,9 +283,8 @@ def test_sporadic_type_is_not_inflated_by_years_present(tmp_path):
def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
# Two postcodes policed by different forces. force-a publishes 2023+2024;
- # force-b publishes only 2023 (a 2024 gap, like Greater Manchester). The
- # b-postcode's headline must pool over force-b's 12 covered months only,
- # and its by-year series must NOT contain a 2024 bar or coverage entry.
+ # force-b publishes only 2023 (a 2024 gap). The b-postcode's raw figure must
+ # pool over force-b's 12 covered months only.
units = tmp_path / "units"
_write_boundaries(
units,
@@ -299,25 +298,21 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
for month in range(1, 13):
ym23 = f"2023-{month:02d}"
ym24 = f"2024-{month:02d}"
- # force-a covers AB1 in both years; one burglary per month in 2024.
_write_month(crime, ym23, [], force="force-a")
_write_month(
crime, ym24, [_crime_row(ym24, 1005, 1005, "Burglary")], force="force-a"
)
- # force-b covers CD1 in 2023 only: one burglary per month.
_write_month(
crime, ym23, [_crime_row(ym23, 9005, 9005, "Burglary")], force="force-b"
)
- avg_df, by_year_df = _run(tmp_path, crime, units)
- rows = {r["postcode"]: r for r in avg_df.to_dicts()}
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units)
+ rows = {r["postcode"]: r for r in raw_df.to_dicts()}
# force-a postcode: 12 burglaries over 24 covered months -> 6/yr.
- assert rows["AB1 1AA"]["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
- # force-b postcode: 12 burglaries over 12 covered months -> 12/yr. Under
- # the old global calendar this would have been diluted to 6/yr by the
- # uncovered 2024.
- assert rows["CD1 1AA"]["Burglary (avg/yr)"] == pytest.approx(12.0, abs=0.05)
+ assert rows["AB1 1AA"][_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
+ # force-b postcode: 12 burglaries over 12 covered months -> 12/yr.
+ assert rows["CD1 1AA"][_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
by_rows = {r["postcode"]: r for r in by_year_df.to_dicts()}
b_coverage = {c["year"]: c["months"] for c in by_rows["CD1 1AA"]["covered_years"]}
@@ -328,59 +323,10 @@ def test_force_gap_years_are_excluded_not_zeroed(tmp_path):
assert a_coverage == {2023: 12, 2024: 12}
-def test_residue_incidents_in_uncovered_years_are_excluded(tmp_path):
- # force-b stops publishing after 2023, but a force-a file contains a 2024
- # incident that falls inside the b-postcode's buffer (cross-border residue,
- # the Greater Manchester pattern). That incident must not produce a 2024
- # bar for the b-postcode, nor leak into its pooled headline.
- units = tmp_path / "units"
- _write_boundaries(
- units,
- {
- "AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)],
- "CD1": [_square_feature("CD1 1AA", 9000, 9000, 9010, 9010)],
- },
- )
-
- crime = tmp_path / "crime"
- for month in range(1, 13):
- ym23 = f"2023-{month:02d}"
- ym24 = f"2024-{month:02d}"
- _write_month(crime, ym23, [], force="force-a")
- # b's own 2023 incidents establish force-b as its home force.
- _write_month(
- crime,
- ym23,
- [_crime_row(ym23, 9005, 9005, "Burglary")] if month <= 6 else [],
- force="force-b",
- )
- # 2024: only force-a publishes; one of its incidents lands in CD1 1AA.
- _write_month(
- crime,
- ym24,
- [_crime_row(ym24, 9005, 9005, "Burglary")] if month == 1 else [],
- force="force-a",
- )
-
- avg_df, by_year_df = _run(tmp_path, crime, units)
-
- b_row = avg_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
- # Pooled over force-b's 12 covered months (2023): 6 incidents -> 6/yr.
- # The residue 2024 incident is excluded (force-b published 0 months in 2024).
- assert b_row["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
-
- b_by = by_year_df.filter(pl.col("postcode") == "CD1 1AA").row(0, named=True)
- bars = {p["year"]: p["count"] for p in b_by["Burglary (by year)"]}
- assert set(bars) == {2023}
- coverage = {c["year"]: c["months"] for c in b_by["covered_years"]}
- assert coverage == {2023: 12}
-
-
def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
# 2023 fully covered; 2024 has only 2 published months. With the default
- # 6-month minimum, 2024 must produce neither a bar (annualising x6 charts
- # noise) nor a coverage entry -- but its incidents and months still count
- # toward the pooled headline.
+ # 6-month minimum, 2024 must produce no bar -- but its incidents and months
+ # still count toward the pooled raw figure.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
@@ -394,12 +340,10 @@ def test_partial_years_below_min_bar_months_get_no_bar(tmp_path):
ym = f"2024-{month:02d}"
_write_month(crime, ym, [_crime_row(ym, 1005, 1005, "Burglary")])
- avg_df, by_year_df = _run(tmp_path, crime, units)
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units)
# Pooled: 14 incidents over 14 covered months -> 12/yr.
- assert avg_df.row(0, named=True)["Burglary (avg/yr)"] == pytest.approx(
- 12.0, abs=0.05
- )
+ assert raw_df.row(0, named=True)[_raw("Burglary")] == pytest.approx(12.0, abs=0.05)
row = by_year_df.row(0, named=True)
bars = {p["year"]: p["count"] for p in row["Burglary (by year)"]}
assert set(bars) == {2023}
@@ -425,52 +369,119 @@ def test_by_year_output_is_dense_with_coverage(tmp_path):
crime = tmp_path / "crime"
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Burglary")])
- avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
assert by_year_df.height == 2
quiet = by_year_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
assert quiet["Burglary (by year)"] is None
assert [c["year"] for c in quiet["covered_years"]] == [2024]
- # And the headline for the quiet postcode is a genuine 0, not null.
- quiet_avg = avg_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
- assert quiet_avg["Burglary (avg/yr)"] == 0.0
+ # The raw figure for the covered, crime-free postcode is a genuine 0, not null.
+ quiet_raw = raw_df.filter(pl.col("postcode") == "AB1 1AB").row(0, named=True)
+ assert quiet_raw[_raw("Burglary")] == 0.0
-def test_serious_rollup_avg_yr_equals_sum_of_components(tmp_path):
- # Burglary only in 2014, Robbery only in 2024 (one incident each, 2 covered
- # months total). Components pool over the same covered window (each
- # 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
+def test_serious_rollup_equals_sum_of_components(tmp_path):
+ # Burglary only in 2023, Robbery only in 2024 (one incident each, 2 covered
+ # months total, both inside the 7-year window). Components pool over the same
+ # covered window (each 1 x 12 / 2 = 6/yr) and the rollup equals their sum.
units = tmp_path / "units"
_write_boundaries(
units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
)
crime = tmp_path / "crime"
- _write_month(crime, "2014-01", [_crime_row("2014-01", 1005, 1005, "Burglary")])
+ _write_month(crime, "2023-01", [_crime_row("2023-01", 1005, 1005, "Burglary")])
_write_month(crime, "2024-01", [_crime_row("2024-01", 1005, 1005, "Robbery")])
- avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
- avg = avg_df.row(0, named=True)
- assert avg["Burglary (avg/yr)"] == pytest.approx(6.0, abs=0.05)
- assert avg["Robbery (avg/yr)"] == pytest.approx(6.0, abs=0.05)
- # Rollup == sum of its component (avg/yr) columns.
- assert avg["Serious crime (avg/yr)"] == pytest.approx(12.0, abs=0.05)
- assert avg["Serious crime (avg/yr)"] == pytest.approx(
- avg["Burglary (avg/yr)"] + avg["Robbery (avg/yr)"], abs=0.05
+ row = raw_df.row(0, named=True)
+ assert row[_raw("Burglary")] == pytest.approx(6.0, abs=0.05)
+ assert row[_raw("Robbery")] == pytest.approx(6.0, abs=0.05)
+ assert row[_raw("Serious crime")] == pytest.approx(12.0, abs=0.05)
+ assert row[_raw("Serious crime")] == pytest.approx(
+ row[_raw("Burglary")] + row[_raw("Robbery")], abs=0.05
)
- # The by-year rollup series remains the per-year sum of the component bars.
serious_bars = {
p["year"]: p["count"]
for p in by_year_df.row(0, named=True)["Serious crime (by year)"]
}
assert serious_bars == {
- 2014: pytest.approx(12.0, abs=0.05),
+ 2023: pytest.approx(12.0, abs=0.05),
2024: pytest.approx(12.0, abs=0.05),
}
+def test_records_capture_each_counted_incident(tmp_path):
+ # Each (incident, postcode) match within the records window becomes a record
+ # row, carrying month/type/location/outcome/coords. A boundary incident
+ # counted for two postcodes appears once per postcode.
+ units = tmp_path / "units"
+ _write_boundaries(
+ units,
+ {
+ "AB1": [
+ _square_feature("AB1 1AA", 1000, 1000, 1010, 1010),
+ _square_feature("AB1 1AB", 1080, 1000, 1090, 1010),
+ ]
+ },
+ )
+ crime = tmp_path / "crime"
+ _write_month(
+ crime,
+ "2024-03",
+ [
+ # In the buffer overlap -> recorded for both postcodes.
+ _crime_row("2024-03", 1045, 1005, "Burglary", location="On or near High St", outcome="Under investigation"),
+ # Only in AB1 1AA's buffer; null outcome (police.uk leaves ASB blank).
+ _crime_row("2024-03", 1005, 1005, "Anti-social behaviour", location="On or near Mill Ln", outcome=""),
+ ],
+ )
+
+ _, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
+
+ assert set(records_df.columns) == {
+ "postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"
+ }
+ # Sorted by postcode.
+ assert records_df["postcode"].is_sorted()
+ # Burglary appears for BOTH postcodes (boundary multiplicity); ASB only for AA.
+ by_pc = records_df.group_by("postcode").agg(pl.col("crime_type").sort())
+ counts = {r["postcode"]: r["crime_type"] for r in by_pc.to_dicts()}
+ assert counts["AB1 1AA"] == ["Anti-social behaviour", "Burglary"]
+ assert counts["AB1 1AB"] == ["Burglary"]
+ # month_index = year*12 + (month-1) for 2024-03.
+ assert set(records_df["month_index"].to_list()) == {2024 * 12 + 2}
+ # Null outcome round-trips as null, not the string "".
+ asb = records_df.filter(pl.col("crime_type") == "Anti-social behaviour").row(0, named=True)
+ assert asb["outcome"] is None
+ assert asb["location"] == "On or near Mill Ln"
+
+
+def test_records_window_aligns_to_the_headline_calendar_window(tmp_path):
+ # Records must cover exactly the longest (7y) headline window, which is
+ # calendar-year based. With a mid-year latest month (2025-06) the 7y window
+ # is calendar years 2019..2025, so an incident in 2018-09 -- which the
+ # headline excludes -- must also be excluded from the records, even though a
+ # naive rolling 84-month span (ending 2025-06) would wrongly include it. The
+ # first month of the earliest window year (2019-01) is kept.
+ units = tmp_path / "units"
+ _write_boundaries(
+ units, {"AB1": [_square_feature("AB1 1AA", 1000, 1000, 1010, 1010)]}
+ )
+ crime = tmp_path / "crime"
+ _write_month(crime, "2018-09", [_crime_row("2018-09", 1005, 1005, "Burglary")])
+ _write_month(crime, "2019-01", [_crime_row("2019-01", 1005, 1005, "Burglary")])
+ _write_month(crime, "2025-06", [_crime_row("2025-06", 1005, 1005, "Burglary")])
+
+ _, _, records_df = _run(tmp_path, crime, units, min_bar_months=1)
+
+ # 2018-09 (year*12+8) is in the rolling 84-month span but NOT the 7y calendar
+ # window, so it is excluded; 2019-01 and 2025-06 are kept.
+ assert set(records_df["month_index"].to_list()) == {2019 * 12 + 0, 2025 * 12 + 5}
+
+
def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
units = tmp_path / "units"
_write_boundaries(
@@ -487,11 +498,10 @@ def test_unknown_crime_type_is_dropped_with_warning(tmp_path, capsys):
],
)
- avg_df, _ = _run(tmp_path, crime, units)
- columns = avg_df.columns
- # The unknown type is dropped (no column for it) but a warning is emitted.
- assert "Cyber fraud (avg/yr)" not in columns
- assert "Burglary (avg/yr)" in columns
+ raw_df, _, _ = _run(tmp_path, crime, units)
+ columns = raw_df.columns
+ assert _raw("Cyber fraud") not in columns
+ assert _raw("Burglary") in columns
err = capsys.readouterr().err
assert "Cyber fraud" in err
assert "WARNING" in err
@@ -515,11 +525,11 @@ def test_legacy_crime_types_are_mapped(tmp_path):
],
)
- avg_df, by_year_df = _run(tmp_path, crime, units, min_bar_months=1)
- row = avg_df.to_dicts()[0]
- # Single postcode -> area-norm factor 1.0; single covered month -> x12.
- assert row["Violence and sexual offences (avg/yr)"] == 12.0
- assert row["Public order (avg/yr)"] == 12.0
+ raw_df, by_year_df, _ = _run(tmp_path, crime, units, min_bar_months=1)
+ row = raw_df.to_dicts()[0]
+ # Single covered month (relative to a 2013-latest window) -> x12.
+ assert row[_raw("Violence and sexual offences")] == 12.0
+ assert row[_raw("Public order")] == 12.0
by_year_row = by_year_df.row(0, named=True)
assert by_year_row["Violence and sexual offences (by year)"] == [
diff --git a/pipeline/transform/test_join_price_estimates.py b/pipeline/transform/test_join_price_estimates.py
new file mode 100644
index 0000000..7a41306
--- /dev/null
+++ b/pipeline/transform/test_join_price_estimates.py
@@ -0,0 +1,136 @@
+"""Tests for joining slim price estimates back onto properties.parquet.
+
+estimate.py emits (Postcode, coalesced address, estimate columns) and
+join_estimates attaches them by that natural key. These tests pin the
+properties that make the key safe: it maps estimates onto the right rows
+regardless of order (a shuffled estimates frame is the worst case), it is
+idempotent, and it refuses a partial/foreign estimates file rather than
+silently nulling prices.
+"""
+
+from pathlib import Path
+
+import polars as pl
+import pytest
+
+from pipeline.transform.join_price_estimates import join_estimates
+from pipeline.transform.price_estimation.utils import (
+ ESTIMATE_COLUMNS,
+ JOIN_ADDRESS,
+ JOIN_KEYS,
+ join_address_expr,
+)
+
+N = 200
+
+
+def _write_merged(path: Path) -> pl.DataFrame:
+ """properties.parquet with the natural-key columns, a sentinel order column,
+ and no estimates. Half the rows are sale-addressed, half EPC-only, so the
+ coalesce in the key is exercised; every coalesced address is unique."""
+ df = pl.DataFrame(
+ {
+ "Postcode": [f"AA{i % 7} {i % 9}AA" for i in range(N)],
+ "Address per Property Register": [
+ f"reg-{i}" if i % 2 == 0 else None for i in range(N)
+ ],
+ "Address per EPC": [f"epc-{i}" if i % 2 == 1 else None for i in range(N)],
+ "order": list(range(N)),
+ "junk": [f"x{i}" for i in range(N)],
+ }
+ )
+ df.write_parquet(path)
+ return df
+
+
+def _write_estimates(path: Path, merged_path: Path, *, shuffle: bool = True) -> None:
+ """Estimates keyed by the natural key, derived from the merged file the way
+ estimate.py does. Estimate = order * 1000 so each row is checkable. Shuffled
+ by default to prove order-independence."""
+ est = (
+ pl.read_parquet(merged_path)
+ .with_columns(join_address_expr())
+ .with_columns(
+ (pl.col("order") * 1000).cast(pl.Float64).alias("Estimated current price"),
+ (pl.col("order") * 10).cast(pl.Int32).alias("Est. price per sqm"),
+ )
+ .select(*JOIN_KEYS, *ESTIMATE_COLUMNS)
+ )
+ if shuffle:
+ est = est.sample(fraction=1.0, shuffle=True, seed=7)
+ est.write_parquet(path)
+
+
+def test_join_attaches_estimates_to_the_right_rows(tmp_path: Path):
+ props = tmp_path / "properties.parquet"
+ estimates = tmp_path / "price_estimates.parquet"
+ _write_merged(props)
+ _write_estimates(estimates, props)
+
+ written = join_estimates(props, estimates)
+ out = pl.read_parquet(props)
+
+ assert written == N
+ assert out.height == N
+ # Order preserved and the address-half of the key is not left behind.
+ assert out["order"].to_list() == list(range(N))
+ assert out["junk"].to_list() == [f"x{i}" for i in range(N)]
+ assert JOIN_ADDRESS not in out.columns
+ # Every row carries its own estimate, matched by key despite the shuffle.
+ assert out["Estimated current price"].to_list() == [float(i * 1000) for i in range(N)]
+ assert out["Est. price per sqm"].to_list() == [i * 10 for i in range(N)]
+ assert out["Estimated current price"].null_count() == 0
+
+
+def test_rerun_is_idempotent(tmp_path: Path):
+ props = tmp_path / "properties.parquet"
+ estimates = tmp_path / "price_estimates.parquet"
+ _write_merged(props)
+ _write_estimates(estimates, props)
+
+ join_estimates(props, estimates)
+ first = pl.read_parquet(props)
+ join_estimates(props, estimates) # second run on the augmented file
+ second = pl.read_parquet(props)
+
+ assert second.equals(first)
+ assert second.columns.count("Estimated current price") == 1
+ assert second.columns.count("Est. price per sqm") == 1
+
+
+def test_missing_estimate_is_rejected(tmp_path: Path):
+ """A property with no matching estimate (diverged dwelling universe) must
+ fail loudly rather than silently leave its price null."""
+ props = tmp_path / "properties.parquet"
+ estimates = tmp_path / "price_estimates.parquet"
+ _write_merged(props)
+ _write_estimates(estimates, props)
+ # Drop one estimate so a property key is no longer covered.
+ pl.read_parquet(estimates).head(N - 1).write_parquet(estimates)
+
+ with pytest.raises(ValueError, match="no matching estimate"):
+ join_estimates(props, estimates)
+
+
+def test_duplicate_key_is_rejected(tmp_path: Path):
+ props = tmp_path / "properties.parquet"
+ estimates = tmp_path / "price_estimates.parquet"
+ _write_merged(props)
+ _write_estimates(estimates, props)
+ # Force row 1's key to collide with row 0's.
+ est = pl.read_parquet(estimates).sort("Estimated current price")
+ row0 = est.row(0, named=True)
+ est = est.with_columns(
+ pl.when(pl.int_range(pl.len()) == 1)
+ .then(pl.lit(row0["Postcode"]))
+ .otherwise(pl.col("Postcode"))
+ .alias("Postcode"),
+ pl.when(pl.int_range(pl.len()) == 1)
+ .then(pl.lit(row0[JOIN_ADDRESS]))
+ .otherwise(pl.col(JOIN_ADDRESS))
+ .alias(JOIN_ADDRESS),
+ )
+ est.write_parquet(estimates)
+
+ with pytest.raises(ValueError, match="not unique"):
+ join_estimates(props, estimates)
diff --git a/pipeline/transform/test_merge.py b/pipeline/transform/test_merge.py
index 234aad0..497735a 100644
--- a/pipeline/transform/test_merge.py
+++ b/pipeline/transform/test_merge.py
@@ -10,14 +10,11 @@ from pipeline.transform.merge import (
LISTED_BUILDING_FEATURE,
TREE_DENSITY_FEATURE,
_LISTING_OVERLAY_SOURCES,
- _active_english_postcode_area,
_build_unmatched_listing_seed_rows,
_canonical_postcode_expr,
_best_listing_match,
_coalesce_direct_epc_columns,
- _dedupe_collapsed_properties,
_fill_property_level_no_defaults,
- _filter_to_active_english_postcodes,
_join_area_side_tables,
_finalize_listings,
_integrate_listings,
@@ -31,7 +28,6 @@ from pipeline.transform.merge import (
_matched_listed_building_flags,
_postcode_conservation_area_flags,
_postcode_listed_building_candidates,
- _remap_terminated_postcodes,
_split_normal_outputs,
_tree_density_by_postcode,
_validate_lad_source_coverage,
@@ -39,6 +35,12 @@ from pipeline.transform.merge import (
_validate_postcode_feature_output,
_validate_property_postcodes,
)
+from pipeline.transform.property_base import (
+ _active_english_postcode_area,
+ _dedupe_collapsed_properties,
+ _filter_to_active_english_postcodes,
+ _remap_terminated_postcodes,
+)
def test_less_deprived_percentile_expr_preserves_direction_and_nulls() -> None:
@@ -115,13 +117,16 @@ def test_tree_density_is_area_level_and_survives_the_split() -> None:
assert TREE_DENSITY_FEATURE not in properties_df.columns
-def test_crime_columns_are_spatial_counts_not_per_capita() -> None:
- # Crime is now a raw spatial count per postcode; the per-1k-residents
- # variants were dropped along with the LSOA population denominator.
- assert "Serious crime (avg/yr)" in _AREA_COLUMNS
- assert "Minor crime (avg/yr)" in _AREA_COLUMNS
- assert "Serious crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
- assert "Minor crime per 1k residents (avg/yr)" not in _AREA_COLUMNS
+def test_crime_columns_are_average_annual_counts() -> None:
+ # Crime is the average annual recorded incident count (incidents/yr) over
+ # 7-year and 2-year windows; the old per-1,000 "(per 1k/yr, …)" rate columns
+ # are gone.
+ assert "Serious crime (/yr, 7y)" in _AREA_COLUMNS
+ assert "Serious crime (/yr, 2y)" in _AREA_COLUMNS
+ assert "Minor crime (/yr, 7y)" in _AREA_COLUMNS
+ assert "Burglary (/yr, 2y)" in _AREA_COLUMNS
+ assert "Serious crime (avg/yr)" not in _AREA_COLUMNS
+ assert "Minor crime (avg/yr)" not in _AREA_COLUMNS
def test_active_english_postcode_area_filters_to_active_england() -> None:
@@ -292,8 +297,8 @@ def test_join_area_side_tables_does_not_fan_out_on_unique_keys() -> None:
crime = pl.LazyFrame(
{
"postcode": ["AA1 1AA", "BB2 2BB"],
- "Serious crime (avg/yr)": [1.0, 2.0],
- "Minor crime (avg/yr)": [3.0, 4.0],
+ "Serious crime (/yr, 7y)": [1.0, 2.0],
+ "Minor crime (/yr, 7y)": [3.0, 4.0],
}
)
joined = _join_area_side_tables(
@@ -343,8 +348,8 @@ def test_join_area_side_tables_normalizes_broadband_postcode_key() -> None:
crime = pl.LazyFrame(
{
"postcode": ["AB1 2CD", "EF3 4GH"],
- "Serious crime (avg/yr)": [1.0, 2.0],
- "Minor crime (avg/yr)": [3.0, 4.0],
+ "Serious crime (/yr, 7y)": [1.0, 2.0],
+ "Minor crime (/yr, 7y)": [3.0, 4.0],
}
)
# AB1 2CD arrives lowercase + un-spaced; EF3 4GH arrives under two distinct
@@ -1314,28 +1319,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
return pl.LazyFrame({"postcode": ["AA1 1AA", "BB2 2BB"], **extra})
# Crime is present only for AA1 1AA; BB2 2BB is absent from the table. The
- # rollup headlines are precomputed values (deliberately NOT the per-type sum,
- # which would be 10.0 each) so this test proves the merge consumes the
- # precomputed column rather than re-summing per-type columns.
+ # rollup rate columns are precomputed in crime_spatial and read straight
+ # through unchanged (the merge no longer renames or re-sums them).
crime = pl.LazyFrame(
{
"postcode": ["AA1 1AA"],
- "Violence and sexual offences (avg/yr)": [1.0],
- "Robbery (avg/yr)": [2.0],
- "Burglary (avg/yr)": [3.0],
- "Possession of weapons (avg/yr)": [4.0],
- "Anti-social behaviour (avg/yr)": [1.0],
- "Criminal damage and arson (avg/yr)": [1.0],
- "Shoplifting (avg/yr)": [1.0],
- "Bicycle theft (avg/yr)": [1.0],
- "Theft from the person (avg/yr)": [1.0],
- "Other theft (avg/yr)": [1.0],
- "Vehicle crime (avg/yr)": [1.0],
- "Public order (avg/yr)": [1.0],
- "Drugs (avg/yr)": [1.0],
- "Other crime (avg/yr)": [1.0],
- "Serious crime (avg/yr)": [7.5],
- "Minor crime (avg/yr)": [4.2],
+ "Burglary (/yr, 7y)": [3.0],
+ "Burglary (/yr, 2y)": [3.5],
+ "Serious crime (/yr, 7y)": [7.5],
+ "Serious crime (/yr, 2y)": [8.0],
+ "Minor crime (/yr, 7y)": [4.2],
+ "Minor crime (/yr, 2y)": [4.6],
}
)
@@ -1364,16 +1358,17 @@ def test_join_area_side_tables_preserves_missing_crime_as_null() -> None:
by_postcode = {
row["postcode"]: row
for row in joined.select(
- "postcode", "serious_crime_avg_yr", "minor_crime_avg_yr"
+ "postcode",
+ "Serious crime (/yr, 7y)",
+ "Minor crime (/yr, 2y)",
).iter_rows(named=True)
}
- # Present postcode: rollups are the precomputed headline values, read through
- # unchanged (NOT the per-type sum of 10.0).
- assert by_postcode["AA1 1AA"]["serious_crime_avg_yr"] == 7.5
- assert by_postcode["AA1 1AA"]["minor_crime_avg_yr"] == 4.2
- # Missing postcode: rollups stay null rather than fabricating 0.0.
- assert by_postcode["BB2 2BB"]["serious_crime_avg_yr"] is None
- assert by_postcode["BB2 2BB"]["minor_crime_avg_yr"] is None
+ # Present postcode: rollup rates pass through unchanged.
+ assert by_postcode["AA1 1AA"]["Serious crime (/yr, 7y)"] == 7.5
+ assert by_postcode["AA1 1AA"]["Minor crime (/yr, 2y)"] == 4.6
+ # Missing postcode: rates stay null rather than fabricating 0.0.
+ assert by_postcode["BB2 2BB"]["Serious crime (/yr, 7y)"] is None
+ assert by_postcode["BB2 2BB"]["Minor crime (/yr, 2y)"] is None
def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
diff --git a/server-rs/src/data.rs b/server-rs/src/data.rs
index d794abd..2994d6b 100644
--- a/server-rs/src/data.rs
+++ b/server-rs/src/data.rs
@@ -1,6 +1,7 @@
mod actual_listings;
pub mod area_crime_averages;
pub mod crime_by_year;
+pub mod crime_records;
mod developments;
mod places;
mod poi;
@@ -65,6 +66,7 @@ where
pub use actual_listings::{ActualListing, ActualListingData};
pub use area_crime_averages::AreaCrimeAverages;
pub use crime_by_year::CrimeByYearData;
+pub use crime_records::CrimeRecords;
pub use developments::{DevelopmentData, DevelopmentSite};
pub use places::{
compute_trigrams, normalize_search_text, place_alias_tokens, trigram_similarity, PlaceData,
diff --git a/server-rs/src/data/area_crime_averages.rs b/server-rs/src/data/area_crime_averages.rs
index 8c6d59e..b3a82e7 100644
--- a/server-rs/src/data/area_crime_averages.rs
+++ b/server-rs/src/data/area_crime_averages.rs
@@ -1,41 +1,61 @@
-//! Precomputed per-outcode and per-postcode-sector average crime rates.
+//! Precomputed per-outcode and per-postcode-sector average crime counts.
//!
//! The right pane shows each crime metric's national average (the global
//! feature-histogram mean). To let users see how an area compares with its
-//! immediate surroundings, we also precompute the mean headline crime rate
-//! (`"X (avg/yr)"`) across every property in the selection's outcode (e.g.
-//! `"E14"`) and postcode sector (e.g. `"E14 2"`).
+//! immediate surroundings, we also show the mean average-annual crime count
+//! (`"X (/yr, 7y|2y)"`) across every property in the selection's outcode
+//! (e.g. `"E14"`) and postcode sector (e.g. `"E14 2"`).
//!
-//! Crime figures are constant within a postcode (the pipeline merges them on
-//! the postcode key), so each postcode's value is read once — from its first
-//! row — and property-weighted by the postcode's row count. That keeps these
-//! averages on the same property-weighted basis as the national average, so the
+//! These averages are precomputed by the data pipeline
+//! (`pipeline/transform/area_crime_averages.py`) and loaded here from a side
+//! parquet. Crime figures are constant within a postcode, so the pipeline
+//! property-weights each postcode's value by its property count — keeping these
+//! averages on the same property-weighted basis as the per-selection mean, so the
//! four numbers (this area / sector / outcode / nation) are directly comparable.
-use rustc_hash::FxHashMap;
+use std::path::Path;
-/// Crime-feature name suffix that marks an annualised headline-rate column
-/// (e.g. `"Burglary (avg/yr)"`). Stripped to derive the bare type name.
-pub const AVG_YR_SUFFIX: &str = " (avg/yr)";
+use anyhow::{bail, Context};
+use polars::prelude::PlRefPath;
+use polars::prelude::*;
+use rustc_hash::FxHashMap;
+use tracing::info;
+
+use super::run_polars_io;
+
+/// Marker that identifies an average-annual crime-count column (e.g.
+/// `"Burglary (/yr, 7y)"`). These are the filterable, area-comparable figures.
+/// The full column name is kept as the key, so each per-area mean aligns with the
+/// feature the frontend requests.
+pub const COUNT_MARKER: &str = " (/yr, ";
+
+/// `scope` column discriminator values written by the pipeline.
+const SCOPE_NATIONAL: &str = "national";
+const SCOPE_OUTCODE: &str = "outcode";
+const SCOPE_SECTOR: &str = "sector";
pub struct AreaCrimeAverages {
- /// Bare crime-type names (suffix stripped, e.g. `"Burglary"`), index-aligned
- /// with the per-area mean vectors. Matches `CrimeYearStats.name`.
+ /// Full crime feature names (e.g. `"Burglary (/yr, 7y)"`), index-aligned with
+ /// the per-area mean vectors. Matches the feature names the frontend requests,
+ /// so each NumberLine can look its average up directly.
pub crime_types: Vec,
- /// National mean headline rate per crime type (index-aligned with
+ /// National mean headline count per crime type (index-aligned with
/// `crime_types`). An EXACT property-weighted mean over every postcode, so it
/// shares a basis with `by_outcode`/`by_sector` and the per-selection mean —
/// unlike the histogram-bin national average, which is biased upward for the
- /// right-skewed crime densities. `NaN` where no postcode has data.
+ /// right-skewed crime counts. `NaN` where no postcode has data.
pub national: Vec,
- /// Outcode (e.g. `"E14"`) → mean headline rate per crime type. `NaN` where
+ /// Outcode (e.g. `"E14"`) → mean headline count per crime type. `NaN` where
/// the outcode has no data for that type.
pub by_outcode: FxHashMap>,
- /// Postcode sector (e.g. `"E14 2"`) → mean headline rate per crime type.
+ /// Postcode sector (e.g. `"E14 2"`) → mean headline count per crime type.
pub by_sector: FxHashMap>,
}
impl AreaCrimeAverages {
+ /// Empty table — used only by the test-only `AppState` builder (the real
+ /// server always loads the precomputed parquet).
+ #[cfg(test)]
pub fn empty() -> Self {
Self {
crime_types: Vec::new(),
@@ -44,4 +64,202 @@ impl AreaCrimeAverages {
by_sector: FxHashMap::default(),
}
}
+
+ pub fn load(path: &Path) -> anyhow::Result {
+ run_polars_io(|| Self::load_inner(path))
+ }
+
+ fn load_inner(path: &Path) -> anyhow::Result {
+ info!("Loading area crime averages from {}", path.display());
+ let pl_path = PlRefPath::try_from_path(path).with_context(|| {
+ format!(
+ "Failed to normalize area-crime-averages parquet path {}",
+ path.display()
+ )
+ })?;
+ let df = LazyFrame::scan_parquet(pl_path, Default::default())
+ .with_context(|| {
+ format!(
+ "Failed to scan area-crime-averages parquet at {}",
+ path.display()
+ )
+ })?
+ .collect()
+ .with_context(|| {
+ format!(
+ "Failed to read area-crime-averages parquet at {}",
+ path.display()
+ )
+ })?;
+
+ // Crime columns are those carrying the count marker; their full names
+ // are kept (in column order) as the keys, so every per-area mean vector is
+ // index-aligned with `crime_types` and matches the requested feature name.
+ let crime_cols: Vec = df
+ .get_column_names()
+ .iter()
+ .filter(|name| name.contains(COUNT_MARKER))
+ .map(|name| name.to_string())
+ .collect();
+ if crime_cols.is_empty() {
+ bail!(
+ "area-crime-averages parquet at {} has no '*{COUNT_MARKER}*' count columns",
+ path.display()
+ );
+ }
+ let crime_types: Vec = crime_cols.clone();
+ let n = crime_cols.len();
+
+ let scope_col = df
+ .column("scope")
+ .context("area-crime-averages parquet missing 'scope' column")?
+ .str()
+ .context("'scope' column is not a string")?;
+ let area_col = df
+ .column("area")
+ .context("area-crime-averages parquet missing 'area' column")?
+ .str()
+ .context("'area' column is not a string")?;
+
+ // Hold the casts alive while we borrow `Float32Chunked` views into them.
+ let casts: Vec = crime_cols
+ .iter()
+ .map(|name| {
+ df.column(name)
+ .and_then(|col| col.cast(&DataType::Float32))
+ .with_context(|| format!("Failed to read crime column '{name}' as f32"))
+ })
+ .collect::>>()?;
+ let crime_views: Vec<&Float32Chunked> = casts
+ .iter()
+ .zip(crime_cols.iter())
+ .map(|(col, name)| {
+ col.f32()
+ .with_context(|| format!("crime column '{name}' is not f32 after cast"))
+ })
+ .collect::>>()?;
+
+ let read_values = |row: usize| -> Vec {
+ crime_views
+ .iter()
+ .map(|view| view.get(row).unwrap_or(f32::NAN))
+ .collect()
+ };
+
+ let mut national: Option> = None;
+ let mut by_outcode: FxHashMap> = FxHashMap::default();
+ let mut by_sector: FxHashMap> = FxHashMap::default();
+
+ for row in 0..df.height() {
+ let scope = scope_col
+ .get(row)
+ .with_context(|| format!("area-crime-averages row {row} has null scope"))?;
+ match scope {
+ SCOPE_NATIONAL => national = Some(read_values(row)),
+ SCOPE_OUTCODE => {
+ if let Some(area) = area_col.get(row) {
+ by_outcode.insert(area.to_string(), read_values(row));
+ }
+ }
+ SCOPE_SECTOR => {
+ if let Some(area) = area_col.get(row) {
+ by_sector.insert(area.to_string(), read_values(row));
+ }
+ }
+ other => bail!("area-crime-averages row {row} has unknown scope '{other}'"),
+ }
+ }
+
+ let national = national.context(
+ "area-crime-averages parquet has no 'national' row; regenerate it with \
+ pipeline.transform.area_crime_averages",
+ )?;
+ if national.len() != n {
+ bail!(
+ "area-crime-averages national row has {} values, expected {n}",
+ national.len()
+ );
+ }
+
+ info!(
+ outcodes = by_outcode.len(),
+ sectors = by_sector.len(),
+ crime_types = crime_types.len(),
+ "Area crime averages loaded"
+ );
+
+ Ok(Self {
+ crime_types,
+ national,
+ by_outcode,
+ by_sector,
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn write_fixture(path: &Path) {
+ // national + one outcode (E14) + one sector (E14 2). Robbery is null for
+ // the outcode to exercise the NaN round-trip.
+ let mut df = df!(
+ "scope" => ["national", "outcode", "sector"],
+ "area" => ["", "E14", "E14 2"],
+ "Burglary (/yr, 7y)" => [8.75f32, 12.5, 12.5],
+ "Robbery (/yr, 7y)" => [Some(1.43f32), None, Some(2.0)],
+ // A non-crime column must be ignored by the loader.
+ "Median age" => [40.0f32, 41.0, 42.0],
+ )
+ .unwrap();
+ let mut file = std::fs::File::create(path).unwrap();
+ ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+ }
+
+ #[test]
+ fn load_round_trips_national_outcode_sector() {
+ let dir = std::env::temp_dir().join(format!("acavg-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("area_crime_averages.parquet");
+ write_fixture(&path);
+
+ let avgs = AreaCrimeAverages::load(&path).unwrap();
+
+ // Crime columns are discovered by the count marker; "Median age" is not.
+ assert_eq!(
+ avgs.crime_types,
+ vec!["Burglary (/yr, 7y)", "Robbery (/yr, 7y)"]
+ );
+ assert_eq!(avgs.national, vec![8.75, 1.43]);
+
+ let e14 = avgs.by_outcode.get("E14").unwrap();
+ assert_eq!(e14[0], 12.5);
+ // The null robbery value becomes NaN, which the consumer drops to None.
+ assert!(e14[1].is_nan());
+
+ let e14_2 = avgs.by_sector.get("E14 2").unwrap();
+ assert_eq!(e14_2, &vec![12.5, 2.0]);
+
+ std::fs::remove_dir_all(&dir).ok();
+ }
+
+ #[test]
+ fn load_rejects_parquet_without_national_row() {
+ let dir = std::env::temp_dir().join(format!("acavg-nonat-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("no_national.parquet");
+ let mut df = df!(
+ "scope" => ["outcode"],
+ "area" => ["E14"],
+ "Burglary (/yr, 7y)" => [12.5f32],
+ )
+ .unwrap();
+ let mut file = std::fs::File::create(&path).unwrap();
+ ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+
+ assert!(AreaCrimeAverages::load(&path).is_err());
+
+ std::fs::remove_dir_all(&dir).ok();
+ }
}
diff --git a/server-rs/src/data/crime_records.rs b/server-rs/src/data/crime_records.rs
new file mode 100644
index 0000000..ff4b898
--- /dev/null
+++ b/server-rs/src/data/crime_records.rs
@@ -0,0 +1,534 @@
+//! Individual police.uk crime records (last 7 years) backing the right pane's
+//! "individual crimes" list and the `/api/crime-records` endpoint.
+//!
+//! This table is enormous — ~500M rows, because each incident is replicated to
+//! every postcode whose buffer covers it (see [`gather`](CrimeRecords::gather)),
+//! so it is NOT held as a `Vec`: each field is a flat columnar
+//! [`SpillVec`] (mmap-backed and kernel-reclaimable when `--spill-dir` is set),
+//! small string fields are dictionary-encoded, and the parquet is pre-sorted by
+//! postcode so each postcode's records are a contiguous `[start, start+count)`
+//! slice located via a CSR-style offset index. Resident RSS is ~0 until records
+//! are actually read.
+//!
+//! At ~500M rows the parquet's string columns (postcode/type/location/outcome)
+//! decode to tens of GB if read whole, so the loader never materialises the
+//! whole `DataFrame`: it streams the file in bounded row-count chunks (only the
+//! row groups overlapping each slice are decoded) and writes each column
+//! straight into its (optionally spilled) backing store via [`SpillVecBuilder`],
+//! keeping the transient footprint to one chunk plus the index maps.
+
+use std::fs::File;
+use std::path::Path;
+
+use anyhow::{bail, Context};
+use lasso::{Rodeo, RodeoReader, Spur};
+use polars::prelude::*;
+use rustc_hash::FxHashMap;
+use tracing::info;
+
+use super::run_polars_io;
+use super::spill::{SpillVec, SpillVecBuilder};
+
+/// Rows decoded per streaming slice. `with_slice` decodes only the row groups
+/// overlapping the slice, so the transient decode is roughly one chunk's worth
+/// (~tens of MB at the writer's ~123k-row groups) instead of the tens-of-GB
+/// whole-file `DataFrame`. The bound's only dependency on file layout is a
+/// reasonable input row-group size, which our pipeline writer produces.
+const CHUNK_ROWS: usize = 2_000_000;
+
+/// A resolved view of one record (strings dereferenced from the dictionaries).
+pub struct CrimeRecordView<'a> {
+ /// `year * 12 + (month - 1)`.
+ pub month_index: u32,
+ pub crime_type: &'a str,
+ pub outcome: Option<&'a str>,
+ pub location: Option<&'a str>,
+ pub lat: f32,
+ pub lon: f32,
+}
+
+pub struct CrimeRecords {
+ month: SpillVec,
+ ctype: SpillVec,
+ outcome: SpillVec,
+ location: SpillVec,
+ lat: SpillVec,
+ lon: SpillVec,
+ /// Dictionary for `ctype` (bare crime type names, e.g. "Burglary").
+ crime_type_dict: Vec,
+ /// Dictionary for `outcome`; index 0 is the empty/unknown sentinel.
+ outcome_dict: Vec,
+ /// Resolver for the interned `location` strings (`""` means withheld).
+ location_resolver: RodeoReader,
+ /// Postcode → `(start, count)` into the columnar arrays (records for a
+ /// postcode are contiguous because the parquet is sorted by postcode).
+ by_postcode: FxHashMap,
+}
+
+impl CrimeRecords {
+ #[cfg(test)]
+ pub fn empty() -> Self {
+ Self {
+ month: SpillVec::owned(Vec::new()),
+ ctype: SpillVec::owned(Vec::new()),
+ outcome: SpillVec::owned(Vec::new()),
+ location: SpillVec::owned(Vec::new()),
+ lat: SpillVec::owned(Vec::new()),
+ lon: SpillVec::owned(Vec::new()),
+ crime_type_dict: Vec::new(),
+ outcome_dict: vec![String::new()],
+ location_resolver: Rodeo::default().into_reader(),
+ by_postcode: FxHashMap::default(),
+ }
+ }
+
+ /// Number of records stored for a postcode (0 if none).
+ pub fn total_for(&self, postcode: &str) -> u32 {
+ self.by_postcode.get(postcode).map_or(0, |&(_, c)| c)
+ }
+
+ /// Resolve a record index to a borrowing view.
+ pub fn view(&self, idx: u32) -> CrimeRecordView<'_> {
+ let i = idx as usize;
+ let outcome_idx = self.outcome[i] as usize;
+ let outcome = self
+ .outcome_dict
+ .get(outcome_idx)
+ .filter(|s| !s.is_empty())
+ .map(String::as_str);
+ let location = self.location_resolver.resolve(&self.location[i]);
+ CrimeRecordView {
+ month_index: self.month[i],
+ crime_type: self
+ .crime_type_dict
+ .get(self.ctype[i] as usize)
+ .map_or("", String::as_str),
+ outcome,
+ location: (!location.is_empty()).then_some(location),
+ lat: self.lat[i],
+ lon: self.lon[i],
+ }
+ }
+
+ /// Record indices across `postcodes`, newest first, optionally restricted to
+ /// months `>= since_month`. These are exactly the incidents counted for the
+ /// selected postcodes — for a single postcode that is its precise incident
+ /// list; for a multi-postcode selection a boundary incident counted for
+ /// several postcodes appears once per postcode, matching the count. We do not
+ /// de-duplicate because police.uk snaps many genuinely distinct incidents
+ /// (especially anti-social behaviour) to the same point/month and provides no
+ /// per-incident id to tell a true duplicate from two real incidents apart.
+ pub fn gather(&self, postcodes: &[&str], since_month: Option) -> Vec {
+ let month = self.month.as_slice();
+ let mut out: Vec = Vec::new();
+ for pc in postcodes {
+ let Some(&(start, count)) = self.by_postcode.get(*pc) else {
+ continue;
+ };
+ for i in start..start + count {
+ if since_month.map_or(true, |s| month[i as usize] >= s) {
+ out.push(i);
+ }
+ }
+ }
+ out.sort_unstable_by(|&a, &b| month[b as usize].cmp(&month[a as usize]));
+ out
+ }
+
+ pub fn load(path: &Path, spill_dir: Option<&Path>) -> anyhow::Result {
+ run_polars_io(|| Self::load_inner(path, spill_dir, CHUNK_ROWS))
+ }
+
+ fn load_inner(
+ path: &Path,
+ spill_dir: Option<&Path>,
+ chunk_rows: usize,
+ ) -> anyhow::Result {
+ // A zero chunk size would loop forever; the public entry point passes the
+ // const, but tests parameterise this to exercise chunk boundaries.
+ let chunk_rows = chunk_rows.max(1);
+ info!("Loading crime records from {}", path.display());
+
+ // Read the footer once for the row count, and keep it to hand to every
+ // per-chunk reader so the 2.9MB metadata is never re-parsed.
+ let metadata = {
+ let file = File::open(path).with_context(|| {
+ format!("Failed to open crime-records parquet at {}", path.display())
+ })?;
+ ParquetReader::new(file)
+ .get_metadata()
+ .with_context(|| {
+ format!("Failed to read crime-records parquet metadata at {}", path.display())
+ })?
+ .clone()
+ };
+ let n = metadata.num_rows;
+ // Record indices are stored as `u32` (and `by_postcode` holds `(start,
+ // count)` as `u32`), so the table must fit in that index space.
+ if n > u32::MAX as usize {
+ bail!("crime-records parquet has {n} rows, exceeding the u32 record-index limit");
+ }
+
+ // Columns that, when spilling, are written straight into mmap-backed files
+ // as we stream — so the ~9GB of columnar data never lands on the heap.
+ let mut month = SpillVecBuilder::::with_len(n, spill_dir, "crime_month")?;
+ let mut ctype = SpillVecBuilder::::with_len(n, spill_dir, "crime_ctype")?;
+ let mut outcome = SpillVecBuilder::::with_len(n, spill_dir, "crime_outcome")?;
+ let mut location = SpillVecBuilder::::with_len(n, spill_dir, "crime_location")?;
+ let mut lat = SpillVecBuilder::::with_len(n, spill_dir, "crime_lat")?;
+ let mut lon = SpillVecBuilder::::with_len(n, spill_dir, "crime_lon")?;
+
+ let mut crime_type_dict: Vec = Vec::new();
+ let mut type_index: FxHashMap = FxHashMap::default();
+ // Outcome index 0 is the empty/unknown sentinel.
+ let mut outcome_dict: Vec = vec![String::new()];
+ let mut outcome_index: FxHashMap = FxHashMap::default();
+ let mut rodeo = Rodeo::default();
+ let empty_spur = rodeo.get_or_intern("");
+
+ let mut by_postcode: FxHashMap = FxHashMap::default();
+ let mut cur_pc: Option = None;
+ let mut cur_start: u32 = 0;
+ // Absolute record index across all chunks; drives both the CSR index and
+ // the column builders' write order.
+ let mut global_row: u32 = 0;
+
+ let columns: Vec = ["postcode", "month_index", "crime_type", "location", "outcome", "lat", "lon"]
+ .iter()
+ .map(|s| s.to_string())
+ .collect();
+
+ let mut offset = 0usize;
+ while offset < n {
+ let len = chunk_rows.min(n - offset);
+ let file = File::open(path).with_context(|| {
+ format!("Failed to open crime-records parquet at {}", path.display())
+ })?;
+ let mut reader = ParquetReader::new(file);
+ reader.set_metadata(metadata.clone());
+ // `with_slice` only decodes the row groups overlapping `[offset,
+ // offset+len)` (the file is memory-mapped, so untouched groups are
+ // never faulted in), capping the transient decode to one chunk.
+ let df = reader
+ .with_columns(Some(columns.clone()))
+ .with_slice(Some((offset, len)))
+ .finish()
+ .with_context(|| {
+ format!(
+ "Failed to read crime-records rows [{offset}, {}) from {}",
+ offset + len,
+ path.display()
+ )
+ })?;
+
+ let postcode_col = df
+ .column("postcode")
+ .context("crime-records parquet missing 'postcode'")?
+ .str()
+ .context("'postcode' is not a string")?;
+ let month_col = df
+ .column("month_index")
+ .context("crime-records parquet missing 'month_index'")?
+ .cast(&DataType::Int32)
+ .context("'month_index' not castable to i32")?;
+ let month_ca = month_col.i32().context("'month_index' is not i32")?;
+ // Months are `year*12 + month0` (~24_000), always positive. A null or
+ // non-positive value means a corrupt parquet; fail loudly rather than
+ // silently clamping it to 0 and later rendering it as "0000-01".
+ if month_ca.null_count() > 0 {
+ bail!("crime-records 'month_index' has null values (corrupt parquet)");
+ }
+ match month_ca.min() {
+ Some(m) if m > 0 => {}
+ _ => bail!("crime-records 'month_index' must be a positive year*12+month index"),
+ }
+ let type_col = df
+ .column("crime_type")
+ .context("crime-records parquet missing 'crime_type'")?
+ .str()
+ .context("'crime_type' is not a string")?;
+ let location_col = df
+ .column("location")
+ .context("crime-records parquet missing 'location'")?
+ .str()
+ .context("'location' is not a string")?;
+ let outcome_col = df
+ .column("outcome")
+ .context("crime-records parquet missing 'outcome'")?
+ .str()
+ .context("'outcome' is not a string")?;
+ let lat_col = df
+ .column("lat")
+ .context("crime-records parquet missing 'lat'")?
+ .cast(&DataType::Float32)?;
+ let lat_ca = lat_col.f32().context("'lat' is not f32")?;
+ let lon_col = df
+ .column("lon")
+ .context("crime-records parquet missing 'lon'")?
+ .cast(&DataType::Float32)?;
+ let lon_ca = lon_col.f32().context("'lon' is not f32")?;
+
+ let height = df.height();
+ for row in 0..height {
+ // CSR index: the parquet is sorted by postcode, so a change in the
+ // postcode value (across chunk boundaries too) closes the previous
+ // run and opens a new one.
+ let pc = postcode_col
+ .get(row)
+ .with_context(|| {
+ format!("crime-records row {} has null postcode", offset + row)
+ })?
+ .trim();
+ if cur_pc.as_deref() != Some(pc) {
+ if let Some(prev) = cur_pc.take() {
+ by_postcode.insert(prev, (cur_start, global_row - cur_start));
+ }
+ cur_pc = Some(pc.to_string());
+ cur_start = global_row;
+ }
+
+ month.push(month_ca.get(row).unwrap() as u32);
+
+ let ty = type_col.get(row).unwrap_or("");
+ let ty_id = match type_index.get(ty) {
+ Some(&id) => id,
+ None => {
+ let id = u8::try_from(crime_type_dict.len())
+ .context("more than 256 distinct crime types")?;
+ crime_type_dict.push(ty.to_string());
+ type_index.insert(ty.to_string(), id);
+ id
+ }
+ };
+ ctype.push(ty_id);
+
+ let oc = outcome_col.get(row).unwrap_or("");
+ let oc_id = if oc.is_empty() {
+ 0
+ } else {
+ match outcome_index.get(oc) {
+ Some(&id) => id,
+ None => {
+ let id = u8::try_from(outcome_dict.len())
+ .context("more than 256 distinct outcomes")?;
+ outcome_dict.push(oc.to_string());
+ outcome_index.insert(oc.to_string(), id);
+ id
+ }
+ }
+ };
+ outcome.push(oc_id);
+
+ let loc = location_col.get(row).unwrap_or("");
+ location.push(if loc.is_empty() {
+ empty_spur
+ } else {
+ rodeo.get_or_intern(loc)
+ });
+
+ lat.push(lat_ca.get(row).unwrap_or(f32::NAN));
+ lon.push(lon_ca.get(row).unwrap_or(f32::NAN));
+
+ global_row += 1;
+ }
+
+ offset += len;
+ }
+ if let Some(prev) = cur_pc.take() {
+ by_postcode.insert(prev, (cur_start, global_row - cur_start));
+ }
+ debug_assert_eq!(global_row as usize, n, "streamed fewer rows than the parquet declares");
+
+ let records = Self {
+ month: month.finish()?,
+ ctype: ctype.finish()?,
+ outcome: outcome.finish()?,
+ location: location.finish()?,
+ lat: lat.finish()?,
+ lon: lon.finish()?,
+ crime_type_dict,
+ outcome_dict,
+ location_resolver: rodeo.into_reader(),
+ by_postcode,
+ };
+
+ info!(
+ records = n,
+ postcodes = records.by_postcode.len(),
+ crime_types = records.crime_type_dict.len(),
+ outcomes = records.outcome_dict.len(),
+ "Crime records loaded"
+ );
+ Ok(records)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn write_fixture(path: &Path) {
+ // Two postcodes, postcode-sorted. AA1 1AA has 3 records across two
+ // months (a null outcome and a null location), BB2 2BB has 1.
+ let mut df = df!(
+ "postcode" => ["AA1 1AA", "AA1 1AA", "AA1 1AA", "BB2 2BB"],
+ "month_index" => [24300i32, 24300, 24290, 24305],
+ "crime_type" => ["Burglary", "Burglary", "Robbery", "Drugs"],
+ "location" => [Some("On or near A St"), Some("On or near A St"), None, Some("On or near B Rd")],
+ "outcome" => [Some("Under investigation"), None, Some("Court result"), None],
+ "lat" => [51.5f32, 51.5, 51.6, 52.0],
+ "lon" => [-0.1f32, -0.1, -0.2, -1.0],
+ )
+ .unwrap();
+ let mut file = std::fs::File::create(path).unwrap();
+ ParquetWriter::new(&mut file).finish(&mut df).unwrap();
+ }
+
+ #[test]
+ fn loads_indexes_and_gathers() {
+ let dir = std::env::temp_dir().join(format!("crimerec-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("records.parquet");
+ write_fixture(&path);
+
+ let recs = CrimeRecords::load(&path, None).unwrap();
+ assert_eq!(recs.total_for("AA1 1AA"), 3);
+ assert_eq!(recs.total_for("BB2 2BB"), 1);
+ assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
+
+ // Newest-first across the two postcodes.
+ let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
+ assert_eq!(all.len(), 4);
+ let months: Vec = all.iter().map(|&i| recs.view(i).month_index).collect();
+ assert_eq!(months, vec![24305, 24300, 24300, 24290]);
+
+ // `since` window filter (keep months >= 24300).
+ assert_eq!(recs.gather(&["AA1 1AA"], Some(24300)).len(), 2);
+
+ // String resolution + null handling.
+ let robbery = all
+ .iter()
+ .map(|&i| recs.view(i))
+ .find(|v| v.crime_type == "Robbery")
+ .unwrap();
+ assert_eq!(robbery.outcome, Some("Court result"));
+ assert_eq!(robbery.location, None); // null location → None
+ // Two records have a null outcome (an AA1 Burglary and the BB2 Drugs).
+ let null_outcomes = all
+ .iter()
+ .map(|&i| recs.view(i))
+ .filter(|v| v.outcome.is_none())
+ .count();
+ assert_eq!(null_outcomes, 2);
+
+ std::fs::remove_dir_all(&dir).ok();
+ }
+
+ /// The CSR per-postcode index and the column builders must compose correctly
+ /// across streaming chunk boundaries — including a postcode run split between
+ /// two chunks. Forces `chunk_rows = 2` over the 4-row fixture so AA1 1AA's
+ /// three records straddle the boundary (rows 0,1 in chunk 0; row 2 in chunk 1)
+ /// and is exercised both heap-backed (no spill) and mmap-backed (spill).
+ #[test]
+ fn streams_across_chunk_boundaries() {
+ let base = std::env::temp_dir().join(format!("crimerec-chunk-{}", std::process::id()));
+ std::fs::create_dir_all(&base).unwrap();
+ let path = base.join("records.parquet");
+ write_fixture(&path);
+ let spill = base.join("spill");
+ std::fs::create_dir_all(&spill).unwrap();
+
+ for spill_dir in [None, Some(spill.as_path())] {
+ let recs = CrimeRecords::load_inner(&path, spill_dir, 2).unwrap();
+ // Counts match regardless of how the runs were split across chunks.
+ assert_eq!(recs.total_for("AA1 1AA"), 3);
+ assert_eq!(recs.total_for("BB2 2BB"), 1);
+ assert_eq!(recs.total_for("ZZ9 9ZZ"), 0);
+
+ // Full gather, newest-first, identical to the single-chunk load.
+ let all = recs.gather(&["AA1 1AA", "BB2 2BB"], None);
+ assert_eq!(all.len(), 4);
+ let months: Vec = all.iter().map(|&i| recs.view(i).month_index).collect();
+ assert_eq!(months, vec![24305, 24300, 24300, 24290]);
+
+ // The run that straddled the boundary still resolves its strings.
+ let robbery = all
+ .iter()
+ .map(|&i| recs.view(i))
+ .find(|v| v.crime_type == "Robbery")
+ .unwrap();
+ assert_eq!(robbery.outcome, Some("Court result"));
+ assert_eq!(robbery.location, None);
+ }
+
+ std::fs::remove_dir_all(&base).ok();
+ }
+
+ /// Peak/resident RSS in MiB from `/proc/self/status` (Linux only).
+ fn rss_mib() -> (f64, f64) {
+ let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
+ let field = |key: &str| -> f64 {
+ status
+ .lines()
+ .find(|l| l.starts_with(key))
+ .and_then(|l| l.split_whitespace().nth(1))
+ .and_then(|kb| kb.parse::().ok())
+ .map_or(0.0, |kb| kb / 1024.0)
+ };
+ (field("VmHWM:"), field("VmRSS:"))
+ }
+
+ /// Manual, real-data smoke test: load the actual ~500M-row parquet and report
+ /// peak RSS, proving the streaming + spill load completes without the
+ /// tens-of-GB `DataFrame` materialisation that OOMed the old `.collect()`.
+ ///
+ /// Run with:
+ /// PPC_REAL_CRIME_RECORDS=/path/to/crime_records.parquet \
+ /// cargo test --bins -- --ignored --nocapture real_crime_records_load_is_bounded
+ #[test]
+ #[ignore = "needs the full crime_records.parquet; run manually"]
+ fn real_crime_records_load_is_bounded() {
+ let path = std::env::var("PPC_REAL_CRIME_RECORDS")
+ .unwrap_or_else(|_| "../property-data/crime_records.parquet".to_string());
+ let path = Path::new(&path);
+ if !path.exists() {
+ eprintln!("skipping: {} not found", path.display());
+ return;
+ }
+ let spill = std::env::var("PPC_REAL_SPILL")
+ .unwrap_or_else(|_| "../.tmp/crime-spill-realtest".to_string());
+ let spill = Path::new(&spill);
+ std::fs::create_dir_all(spill).unwrap();
+
+ let (hwm_before, _rss_before) = rss_mib();
+ let start = std::time::Instant::now();
+ let recs = CrimeRecords::load(path, Some(spill)).unwrap();
+ let elapsed = start.elapsed();
+ let (hwm_after, rss_after) = rss_mib();
+
+ let total: u64 = recs.by_postcode.values().map(|&(_, c)| c as u64).sum();
+ eprintln!(
+ "loaded {} records across {} postcodes in {:.1}s | RSS peak {:.0}->{:.0} MiB (Δ{:.0}) resident now {:.0} MiB",
+ total,
+ recs.by_postcode.len(),
+ elapsed.as_secs_f64(),
+ hwm_before,
+ hwm_after,
+ hwm_after - hwm_before,
+ rss_after,
+ );
+
+ assert!(recs.by_postcode.len() > 0, "expected at least one postcode");
+ assert!(total > 0, "expected at least one record");
+ // The old `.collect()` decoded all rows' string columns at once (tens of
+ // GB). Streaming must keep the peak growth far below that; a generous 20GiB
+ // ceiling still proves we never materialise the whole file.
+ assert!(
+ hwm_after - hwm_before < 20_480.0,
+ "peak RSS grew by {:.0} MiB during load — streaming/spill not bounding memory",
+ hwm_after - hwm_before
+ );
+
+ std::fs::remove_dir_all(spill).ok();
+ }
+}
diff --git a/server-rs/src/data/property/mod.rs b/server-rs/src/data/property/mod.rs
index b6f96c2..8a3d20f 100644
--- a/server-rs/src/data/property/mod.rs
+++ b/server-rs/src/data/property/mod.rs
@@ -26,9 +26,7 @@ use rustc_hash::FxHashMap;
use serde::Serialize;
use crate::consts::NAN_U16;
-use crate::data::area_crime_averages::{AreaCrimeAverages, AVG_YR_SUFFIX};
use crate::data::spill::SpillVec;
-use crate::utils::{postcode_outcode, postcode_sector};
#[derive(Serialize, Clone)]
pub struct RenovationEvent {
@@ -226,109 +224,6 @@ impl PropertyData {
num_numeric: self.num_numeric,
}
}
-
- /// Precompute mean headline crime rates nationally and per outcode / postcode
- /// sector.
- ///
- /// Crime values are identical for every property in a postcode (the pipeline
- /// merges them on the postcode key), so each postcode is sampled once from
- /// its first row and property-weighted by its row count. All three scopes use
- /// the same exact property-weighted estimator over the same row universe as
- /// the per-selection mean, so the four numbers shown in a crime row (this
- /// selection / sector / outcode / nation) are directly comparable — without
- /// the upward bias of the histogram-bin national average.
- pub fn compute_area_crime_averages(&self) -> AreaCrimeAverages {
- // Crime headline columns are exactly the " (avg/yr)" features.
- let crime_indices: Vec = self
- .feature_names
- .iter()
- .enumerate()
- .filter(|(_, name)| name.ends_with(AVG_YR_SUFFIX))
- .map(|(idx, _)| idx)
- .collect();
- if crime_indices.is_empty() {
- return AreaCrimeAverages::empty();
- }
- let crime_types: Vec = crime_indices
- .iter()
- .map(|&idx| {
- self.feature_names[idx]
- .strip_suffix(AVG_YR_SUFFIX)
- .unwrap_or(&self.feature_names[idx])
- .to_string()
- })
- .collect();
- let n = crime_indices.len();
-
- // (weighted value sum, weight) accumulators per crime type.
- let mut nat_sums = vec![0.0f64; n];
- let mut nat_weights = vec![0u64; n];
- let mut out_acc: FxHashMap, Vec)> = FxHashMap::default();
- let mut sec_acc: FxHashMap, Vec)> = FxHashMap::default();
-
- for (key, rows) in &self.postcode_row_index {
- let Some(&first) = rows.first() else { continue };
- let count = rows.len() as u64;
- let postcode = self.postcode_interner.resolve(key);
- let outcode = postcode_outcode(postcode);
- let sector = postcode_sector(postcode);
-
- for (j, &fi) in crime_indices.iter().enumerate() {
- // A NaN value is "no crime data for this postcode" — skip it so
- // it dilutes neither the sum nor the weight (a genuine gap, not
- // a zero), exactly as the global histogram excludes it.
- let value = self.get_feature(first as usize, fi);
- if !value.is_finite() {
- continue;
- }
- let weighted = value as f64 * count as f64;
- // National counts every postcode (the population the global mean
- // is built over); outcode/sector only when the postcode parses.
- nat_sums[j] += weighted;
- nat_weights[j] += count;
- if let Some(outcode) = outcode {
- let acc = out_acc
- .entry(outcode.to_string())
- .or_insert_with(|| (vec![0.0; n], vec![0; n]));
- acc.0[j] += weighted;
- acc.1[j] += count;
- }
- if let Some(sector) = sector {
- let acc = sec_acc
- .entry(sector.to_string())
- .or_insert_with(|| (vec![0.0; n], vec![0; n]));
- acc.0[j] += weighted;
- acc.1[j] += count;
- }
- }
- }
-
- let means_of = |sums: &[f64], weights: &[u64]| -> Vec {
- sums.iter()
- .zip(weights.iter())
- .map(|(&sum, &weight)| {
- if weight == 0 {
- f32::NAN
- } else {
- (sum / weight as f64) as f32
- }
- })
- .collect()
- };
- let finalize =
- |acc: FxHashMap, Vec)>| -> FxHashMap> {
- acc.into_iter()
- .map(|(area, (sums, weights))| (area, means_of(&sums, &weights)))
- .collect()
- };
-
- AreaCrimeAverages {
- crime_types,
- national: means_of(&nat_sums, &nat_weights),
- by_outcode: finalize(out_acc),
- by_sector: finalize(sec_acc),
- }
- }
}
#[cfg(test)]
diff --git a/server-rs/src/data/spill.rs b/server-rs/src/data/spill.rs
index 977031f..5468d31 100644
--- a/server-rs/src/data/spill.rs
+++ b/server-rs/src/data/spill.rs
@@ -133,6 +133,138 @@ impl SpillVec {
}
}
+/// Builds a [`SpillVec`] incrementally by `push`ing elements one at a time,
+/// when the final length is known up front but the values arrive in a stream.
+///
+/// When a spill `dir` is set (and the length is non-zero) the backing store is a
+/// pre-sized, memory-mapped file and each pushed element is written straight into
+/// it — so the array never exists as a heap `Vec` and is never copied a second
+/// time on finalisation, unlike [`SpillVec::maybe_spill`], which takes an
+/// already-built `Vec` and so needs the whole thing resident first. Without a
+/// spill dir it accumulates into an owned `Vec` (production behaviour, identical
+/// resident cost to the old `Vec::with_capacity` + `maybe_spill`). This is what
+/// lets the ~500M-row crime-records columns load without a multi-GB heap spike.
+///
+/// Exactly `len` elements must be pushed: pushing more panics, and finishing with
+/// fewer is an error.
+pub struct SpillVecBuilder {
+ backing: Builder,
+}
+
+enum Builder {
+ Owned { values: Vec, len: usize },
+ Mapped {
+ map: MmapMut,
+ len: usize,
+ cursor: usize,
+ label: &'static str,
+ _marker: PhantomData,
+ },
+}
+
+impl SpillVecBuilder {
+ /// Create a builder for exactly `len` elements. Spills to `dir` when it is set
+ /// and `len > 0` (a zero-length mmap is invalid); otherwise reserves an owned
+ /// `Vec`. `label` names the backing file for diagnostics.
+ pub fn with_len(len: usize, dir: Option<&Path>, label: &'static str) -> anyhow::Result {
+ match dir {
+ Some(dir) if len > 0 => {
+ let byte_len = len * std::mem::size_of::();
+ let file = anon_file(dir, label)?;
+ allocate_spill_file(&file, byte_len, label)?;
+ // SAFETY: `file` is a freshly-created, exclusively-owned regular
+ // file sized to exactly `byte_len`; no other mapping aliases it.
+ let map = unsafe { MmapMut::map_mut(&file) }.with_context(|| {
+ format!("mapping spill file for '{label}' ({byte_len} bytes)")
+ })?;
+ Ok(Self {
+ backing: Builder::Mapped {
+ map,
+ len,
+ cursor: 0,
+ label,
+ _marker: PhantomData,
+ },
+ })
+ }
+ _ => Ok(Self {
+ backing: Builder::Owned {
+ values: Vec::with_capacity(len),
+ len,
+ },
+ }),
+ }
+ }
+
+ /// Append one element. Panics if more than the declared `len` elements are
+ /// pushed — enforced identically on both backings so a streaming bug fails
+ /// the same way in production (owned) as in dev (spilled).
+ #[inline]
+ pub fn push(&mut self, value: T) {
+ match &mut self.backing {
+ Builder::Owned { values, len } => {
+ assert!(
+ values.len() < *len,
+ "SpillVecBuilder overflow: pushed more than {len} elements"
+ );
+ values.push(value);
+ }
+ Builder::Mapped {
+ map, len, cursor, ..
+ } => {
+ assert!(
+ *cursor < *len,
+ "SpillVecBuilder overflow: pushed more than {len} elements"
+ );
+ // SAFETY: an mmap base is page-aligned (hence aligned for `T`); the
+ // mapping holds exactly `len * size_of::()` bytes and `cursor <
+ // len`, so this writes one in-bounds, aligned `T`. `T: SpillElem`
+ // is `Copy` and padding-free, so the stored byte image is fully
+ // defined and reads back as the same value.
+ unsafe { map.as_mut_ptr().cast::().add(*cursor).write(value) };
+ *cursor += 1;
+ }
+ }
+ }
+
+ /// Seal the builder into a read-only [`SpillVec`]. Exactly the declared `len`
+ /// elements must have been pushed, on both backings.
+ pub fn finish(self) -> anyhow::Result> {
+ match self.backing {
+ Builder::Owned { values, len } => {
+ if values.len() != len {
+ anyhow::bail!(
+ "spill builder finished with {} of {len} elements written",
+ values.len()
+ );
+ }
+ Ok(SpillVec::Owned(values))
+ }
+ Builder::Mapped {
+ map,
+ len,
+ cursor,
+ label,
+ ..
+ } => {
+ if cursor != len {
+ anyhow::bail!(
+ "spill builder for '{label}' finished with {cursor} of {len} elements written"
+ );
+ }
+ let map = map
+ .make_read_only()
+ .with_context(|| format!("sealing spill file for '{label}'"))?;
+ Ok(SpillVec::Mapped {
+ map,
+ len,
+ _marker: PhantomData,
+ })
+ }
+ }
+ }
+}
+
impl std::ops::Deref for SpillVec {
type Target = [T];
@@ -327,4 +459,105 @@ mod tests {
assert!(matches!(mapped, SpillVec::Owned(_)));
assert!(mapped.is_empty());
}
+
+ #[test]
+ fn builder_streams_identically_owned_and_mapped() {
+ let values: Vec = (0..50_000u32).map(|n| n.wrapping_mul(2_246_822_519)).collect();
+
+ // Owned path (no spill dir): pushes accumulate into a heap Vec.
+ let mut owned = SpillVecBuilder::::with_len(values.len(), None, "u32_owned").unwrap();
+ for &v in &values {
+ owned.push(v);
+ }
+ let owned = owned.finish().unwrap();
+ assert!(matches!(owned, SpillVec::Owned(_)));
+ assert_eq!(&*owned, values.as_slice());
+
+ // Mapped path (spill dir): pushes write straight into the mmap, no heap copy.
+ let dir = TempDir::new("builder-u32");
+ let mut mapped =
+ SpillVecBuilder::::with_len(values.len(), Some(dir.path()), "u32_mapped").unwrap();
+ for &v in &values {
+ mapped.push(v);
+ }
+ let mapped = mapped.finish().unwrap();
+ assert!(matches!(mapped, SpillVec::Mapped { .. }));
+ // The mmap-backed slice must be byte-identical to the streamed input.
+ assert_eq!(&*mapped, values.as_slice());
+ }
+
+ #[test]
+ fn builder_spurs_survive_the_mmap_roundtrip() {
+ // Spur is a NonZeroU32 niche type — exercises the streamed write path for
+ // the crime-records `location` column.
+ let mut rodeo = lasso::Rodeo::default();
+ let keys: Vec = (0..3000)
+ .map(|n| rodeo.get_or_intern(format!("loc-{n}")))
+ .collect();
+
+ let dir = TempDir::new("builder-spur");
+ let mut builder =
+ SpillVecBuilder::::with_len(keys.len(), Some(dir.path()), "spurs").unwrap();
+ for &k in &keys {
+ builder.push(k);
+ }
+ let mapped = builder.finish().unwrap();
+ assert!(matches!(mapped, SpillVec::Mapped { .. }));
+ assert_eq!(&*mapped, keys.as_slice());
+ let reader = rodeo.into_resolver();
+ for (idx, &key) in mapped.iter().enumerate() {
+ assert_eq!(reader.resolve(&key), format!("loc-{idx}"));
+ }
+ }
+
+ #[test]
+ fn builder_zero_len_stays_owned() {
+ let dir = TempDir::new("builder-empty");
+ let builder = SpillVecBuilder::::with_len(0, Some(dir.path()), "empty").unwrap();
+ let v = builder.finish().unwrap();
+ // A zero-length mmap is invalid, so empties fall back to an owned Vec.
+ assert!(matches!(v, SpillVec::Owned(_)));
+ assert!(v.is_empty());
+ }
+
+ #[test]
+ fn builder_underfill_is_an_error() {
+ let dir = TempDir::new("builder-underfill");
+ let mut builder =
+ SpillVecBuilder::::with_len(4, Some(dir.path()), "underfill").unwrap();
+ builder.push(1);
+ builder.push(2);
+ // Sealing a spilling builder before all declared elements are written fails
+ // rather than exposing uninitialised mmap tail bytes as valid data.
+ assert!(builder.finish().is_err());
+ }
+
+ #[test]
+ #[should_panic(expected = "overflow")]
+ fn builder_overfill_panics() {
+ let dir = TempDir::new("builder-overfill");
+ let mut builder = SpillVecBuilder::::with_len(2, Some(dir.path()), "overfill").unwrap();
+ builder.push(1);
+ builder.push(2);
+ builder.push(3); // one past the declared length
+ }
+
+ #[test]
+ fn builder_owned_underfill_is_an_error() {
+ // The owned (no-spill, production) path enforces the declared length too,
+ // so a streaming bug can't silently yield a short array in release builds.
+ let mut builder = SpillVecBuilder::::with_len(4, None, "owned-underfill").unwrap();
+ builder.push(1);
+ builder.push(2);
+ assert!(builder.finish().is_err());
+ }
+
+ #[test]
+ #[should_panic(expected = "overflow")]
+ fn builder_owned_overfill_panics() {
+ let mut builder = SpillVecBuilder::::with_len(2, None, "owned-overfill").unwrap();
+ builder.push(1);
+ builder.push(2);
+ builder.push(3); // one past the declared length
+ }
}
diff --git a/server-rs/src/features.rs b/server-rs/src/features.rs
index 9e229c8..a88e3f9 100644
--- a/server-rs/src/features.rs
+++ b/server-rs/src/features.rs
@@ -68,6 +68,59 @@ pub struct FeatureGroup {
pub features: &'static [Feature],
}
+/// Expand each crime type into its two filterable features: a 7-year and a
+/// 2-year window. Each is the average number of recorded incidents per year (the
+/// raw, absolute count — no per-area or per-capita normalisation). The names must
+/// match the `"{type} (/yr, 7y|2y)"` columns written by `crime_spatial`. The
+/// per-incident records are NOT a feature (they are a display-only side table the
+/// server loads directly), so they never appear here and are not filterable.
+macro_rules! crime_features {
+ ($( ($base:literal, $blurb:literal) ),+ $(,)?) => {
+ &[ $(
+ Feature::Numeric(FeatureConfig {
+ name: concat!($base, " (/yr, 7y)"),
+ bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
+ step: 0.1,
+ description: concat!($blurb, " — average recorded incidents per year (last 7 years)"),
+ detail: concat!(
+ $blurb,
+ ", as the average number of recorded incidents per year, over the last \
+ 7 years. Counted from police.uk street-level crime points (anonymised, \
+ snapped to nearby map points) that fall near the postcode boundary — \
+ the raw, absolute count, with no per-area or per-capita adjustment. \
+ Computed over the months the local police force actually published; \
+ known force gaps (e.g. Greater Manchester since mid-2019) are excluded, \
+ not counted as zero crime."
+ ),
+ source: "crime",
+ prefix: "",
+ suffix: "",
+ raw: false,
+ absolute: false,
+ }),
+ Feature::Numeric(FeatureConfig {
+ name: concat!($base, " (/yr, 2y)"),
+ bounds: Bounds::Percentile { low: 2.0, high: 98.0 },
+ step: 0.1,
+ description: concat!($blurb, " — average recorded incidents per year (last 2 years)"),
+ detail: concat!(
+ $blurb,
+ ", as the average number of recorded incidents per year, over the last \
+ 2 years — a more recent but noisier window than the 7-year figure. From \
+ police.uk street-level crime points near the postcode boundary (the raw, \
+ absolute count), over the months the local force published (gaps \
+ excluded, not zeroed)."
+ ),
+ source: "crime",
+ prefix: "",
+ suffix: "",
+ raw: false,
+ absolute: false,
+ }),
+ )+ ]
+ };
+}
+
pub static FEATURE_GROUPS: &[FeatureGroup] = &[
FeatureGroup {
name: "Properties",
@@ -472,247 +525,41 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
FeatureGroup {
name: "Crime",
- features: &[
- Feature::Numeric(FeatureConfig {
- name: "Serious crime (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Relative density of serious crime categories near the postcode",
- detail: "Combined density of violence, robbery, burglary, and weapons possession near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Minor crime (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Relative density of minor crime categories near the postcode",
- detail: "Combined density of anti-social behaviour, shoplifting, bicycle theft, and other lower-severity crime near the postcode, counted from police.uk street-level crime points (anonymised, snapped to nearby map points). This is an area-normalised incident density for the surrounding streets, not a count of incidents per year and not a per-resident risk: busy commercial centres rank high however few people live there. It is normalised to a median-sized catchment so areas are comparable, and computed over the months the local police force actually published data; known force gaps (e.g. Greater Manchester since mid-2019) are excluded rather than counted as zero crime.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Violence and sexual offences (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly violent and sexual offences in the area",
- detail: "Average number of violence and sexual offences per year near the postcode, from police.uk street-level crime data. Includes assault, harassment, and sexual offences.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Burglary (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly burglary offences in the area",
- detail: "Average number of burglary offences per year near the postcode, from police.uk street-level crime data. Includes residential and commercial burglary.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Robbery (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly robbery offences in the area",
- detail: "Average number of robbery offences per year near the postcode, from police.uk street-level crime data. Robbery involves theft with force or threat of force.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Vehicle crime (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly vehicle crime in the area",
- detail: "Average number of vehicle crime incidents per year near the postcode, from police.uk street-level crime data. Includes theft of and from vehicles.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Anti-social behaviour (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly anti-social behaviour incidents in the area",
- detail: "Average number of anti-social behaviour incidents per year near the postcode, from police.uk street-level crime data. Includes nuisance, environmental, and personal anti-social behaviour.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Criminal damage and arson (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly criminal damage and arson in the area",
- detail: "Average number of criminal damage and arson incidents per year near the postcode, from police.uk street-level crime data.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Other theft (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly other theft offences in the area",
- detail: "Average number of 'other theft' offences per year near the postcode, from police.uk street-level crime data. Includes theft not classified under burglary, vehicle crime, shoplifting, or bicycle theft.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Theft from the person (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly theft from the person in the area",
- detail: "Average number of theft from the person offences per year near the postcode, from police.uk street-level crime data. Includes pickpocketing and bag snatching without force.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Shoplifting (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly shoplifting offences in the area",
- detail: "Average number of shoplifting offences per year near the postcode, from police.uk street-level crime data.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Bicycle theft (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly bicycle theft in the area",
- detail: "Average number of bicycle theft offences per year near the postcode, from police.uk street-level crime data.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Drugs (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly drug offences in the area",
- detail: "Average number of drug offences per year near the postcode, from police.uk street-level crime data. Includes possession and trafficking offences.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Possession of weapons (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly weapons possession offences in the area",
- detail: "Average number of possession of weapons offences per year near the postcode, from police.uk street-level crime data.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Public order (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly public order offences in the area",
- detail: "Average number of public order offences per year near the postcode, from police.uk street-level crime data. Includes causing fear, alarm, or distress.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
- Feature::Numeric(FeatureConfig {
- name: "Other crime (avg/yr)",
- bounds: Bounds::Percentile {
- low: 2.0,
- high: 98.0,
- },
- step: 1.0,
- description: "Average yearly other crime in the area",
- detail: "Average number of other crime offences per year near the postcode, from police.uk street-level crime data. A catch-all category for offences not classified elsewhere.",
- source: "crime",
- prefix: "",
- suffix: "",
- raw: false,
- absolute: false,
- }),
+ features: crime_features![
+ (
+ "Serious crime",
+ "Serious crime — violence, robbery, burglary and weapons possession — near the postcode"
+ ),
+ (
+ "Minor crime",
+ "Lower-severity crime — anti-social behaviour, theft, criminal damage, drugs and public order — near the postcode"
+ ),
+ (
+ "Violence and sexual offences",
+ "Violence and sexual offences (assault, harassment, sexual offences) near the postcode"
+ ),
+ ("Burglary", "Burglary (residential and commercial) near the postcode"),
+ ("Robbery", "Robbery (theft with force or threat) near the postcode"),
+ ("Vehicle crime", "Vehicle crime (theft of and from vehicles) near the postcode"),
+ ("Anti-social behaviour", "Anti-social behaviour near the postcode"),
+ ("Criminal damage and arson", "Criminal damage and arson near the postcode"),
+ (
+ "Other theft",
+ "Other theft (not burglary, vehicle, shoplifting or bicycle theft) near the postcode"
+ ),
+ (
+ "Theft from the person",
+ "Theft from the person (pickpocketing, bag snatching) near the postcode"
+ ),
+ ("Shoplifting", "Shoplifting near the postcode"),
+ ("Bicycle theft", "Bicycle theft near the postcode"),
+ ("Drugs", "Drug offences (possession and trafficking) near the postcode"),
+ ("Possession of weapons", "Possession of weapons near the postcode"),
+ (
+ "Public order",
+ "Public order offences (causing fear, alarm or distress) near the postcode"
+ ),
+ ("Other crime", "Other crime (offences not classified elsewhere) near the postcode"),
],
},
FeatureGroup {
@@ -826,9 +673,10 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
// shares sum to ~100% per neighbourhood (LSOA) and render as a
// stacked composition (see STACKED_GROUPS["Neighbours"] in the
// frontend), like the ethnicity, qualifications and vote-share bars.
- // Unlike those, the three shares are ALSO offered as individual
- // filters (they are not added to the display-only skip-list in
- // Filters.tsx), so users can target e.g. owner-occupier-heavy areas.
+ // Unlike those — each folded into a single dropdown filter that
+ // selects one band — the three tenure shares are offered as
+ // individual filters, so users can target e.g. owner-occupier-heavy
+ // areas.
Feature::Numeric(FeatureConfig {
name: "% Owner occupied",
bounds: Bounds::Fixed { min: 0.0, max: 100.0 },
@@ -1292,7 +1140,7 @@ mod tests {
"Income Score", // 0..100 percentile
"% White", // 0..100 percentage
"Noise (dB)", // 50..80, range > threshold
- "Serious crime (avg/yr)", // Percentile bounds, fractional
+ "Serious crime (/yr, 7y)", // Percentile bounds, fractional
"Interior height (m)", // step 0.1
"Estimated current price", // step 10000
] {
diff --git a/server-rs/src/main.rs b/server-rs/src/main.rs
index c804b81..7e9decc 100644
--- a/server-rs/src/main.rs
+++ b/server-rs/src/main.rs
@@ -334,6 +334,18 @@ struct Cli {
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
crime_by_year_path: PathBuf,
+ /// Path to the per-incident crime-records parquet (last 7 years, postcode-
+ /// sorted) backing the "individual crimes" list. Spilled to disk when
+ /// `--spill-dir` is set.
+ #[arg(long, env = "CRIME_RECORDS_PATH")]
+ crime_records_path: PathBuf,
+
+ /// Path to the precomputed national/per-outcode/per-sector crime-averages
+ /// parquet (built by pipeline.transform.area_crime_averages). The right pane
+ /// uses it to compare a selection's crime rates against its surroundings.
+ #[arg(long, env = "AREA_CRIME_AVERAGES_PATH")]
+ area_crime_averages_path: PathBuf,
+
/// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
/// residents; display-only side table for the right pane). Optional: when
/// absent or missing, the area pane simply omits the population figure.
@@ -725,6 +737,16 @@ async fn main() -> anyhow::Result<()> {
Arc::new(data)
};
+ let crime_records = {
+ let path = &cli.crime_records_path;
+ if !path.exists() {
+ bail!("Crime-records parquet not found: {}", path.display());
+ }
+ let data = data::CrimeRecords::load(path, spill_dir)?;
+ trim_allocator("crime-records load");
+ Arc::new(data)
+ };
+
let population = match &cli.population_path {
Some(path) if path.exists() => {
let data = data::PostcodePopulation::load(path)?;
@@ -742,13 +764,11 @@ async fn main() -> anyhow::Result<()> {
};
let area_crime_averages = {
- let data = property_data.compute_area_crime_averages();
- info!(
- outcodes = data.by_outcode.len(),
- sectors = data.by_sector.len(),
- crime_types = data.crime_types.len(),
- "Per-outcode/sector crime averages computed"
- );
+ let path = &cli.area_crime_averages_path;
+ if !path.exists() {
+ bail!("Area-crime-averages parquet not found: {}", path.display());
+ }
+ let data = data::AreaCrimeAverages::load(path)?;
trim_allocator("area crime averages");
Arc::new(data)
};
@@ -781,6 +801,7 @@ async fn main() -> anyhow::Result<()> {
actual_listings,
developments,
crime_by_year,
+ crime_records,
population,
area_crime_averages,
token_cache,
@@ -899,6 +920,10 @@ async fn main() -> anyhow::Result<()> {
"/api/postcode-properties",
get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
)
+ .route(
+ "/api/crime-records",
+ get(routes::get_crime_records).layer(ConcurrencyLimitLayer::new(5)),
+ )
.route(
"/api/screenshot",
get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),
diff --git a/server-rs/src/pocketbase.rs b/server-rs/src/pocketbase.rs
index 56dade1..0c36df8 100644
--- a/server-rs/src/pocketbase.rs
+++ b/server-rs/src/pocketbase.rs
@@ -1319,6 +1319,49 @@ pub async fn ensure_collections(
ensure_autodate_fields(client, base_url, &token, "ai_query_logs").await?;
}
+ // Per-user record of which property listings a user has opened, so visited listings
+ // can be drawn in a distinct colour on the map. One row per (user, url); the unique
+ // index makes re-clicks idempotent.
+ let clicked_listings_index =
+ "CREATE UNIQUE INDEX idx_clicked_listings_user_url ON clicked_listings (user, url)";
+ if !existing.iter().any(|n| n == "clicked_listings") {
+ let users_id = find_users_collection_id(client, base_url, &token).await?;
+ let user_only = Some("user = @request.auth.id".to_string());
+ create_collection(
+ client,
+ base_url,
+ &token,
+ CreateCollection {
+ name: "clicked_listings".to_string(),
+ r#type: "base".to_string(),
+ fields: vec![
+ Field::relation("user", &users_id),
+ Field::text("url", true),
+ Field::autodate("created", true, false),
+ Field::autodate("updated", true, true),
+ ],
+ list_rule: user_only.clone(),
+ view_rule: user_only.clone(),
+ create_rule: user_only.clone(),
+ update_rule: user_only.clone(),
+ delete_rule: user_only,
+ indexes: vec![clicked_listings_index.to_string()],
+ },
+ )
+ .await?;
+ } else {
+ ensure_user_owned_rules(client, base_url, &token, "clicked_listings").await?;
+ ensure_autodate_fields(client, base_url, &token, "clicked_listings").await?;
+ ensure_collection_indexes(
+ client,
+ base_url,
+ &token,
+ "clicked_listings",
+ &[("idx_clicked_listings_user_url", clicked_listings_index)],
+ )
+ .await?;
+ }
+
Ok(())
}
diff --git a/server-rs/src/routes.rs b/server-rs/src/routes.rs
index efcf3d4..8e8bfe6 100644
--- a/server-rs/src/routes.rs
+++ b/server-rs/src/routes.rs
@@ -1,6 +1,7 @@
mod actual_listings;
mod ai_filters;
mod checkout;
+mod crime_records;
mod developments;
mod export;
mod features;
@@ -35,6 +36,7 @@ pub(crate) mod travel_time;
pub use actual_listings::get_actual_listings;
pub use ai_filters::{build_system_prompt, post_ai_filters};
pub use checkout::post_checkout;
+pub use crime_records::get_crime_records;
pub use developments::get_developments;
pub use export::get_export;
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
diff --git a/server-rs/src/routes/ai_filters/parsing.rs b/server-rs/src/routes/ai_filters/parsing.rs
index 2ab5044..ff19cc7 100644
--- a/server-rs/src/routes/ai_filters/parsing.rs
+++ b/server-rs/src/routes/ai_filters/parsing.rs
@@ -336,8 +336,8 @@ mod tests {
"Good+ primary school catchments"
);
assert_eq!(
- canonical_filter_name("Specific crimes:Burglary%20%28avg%2Fyr%29:1"),
- "Burglary (avg/yr)"
+ canonical_filter_name("Specific crimes:Burglary%20%28%2Fyr%2C%207y%29:1"),
+ "Burglary (/yr, 7y)"
);
assert_eq!(
canonical_filter_name("Political vote share:%25%20Labour:0"),
diff --git a/server-rs/src/routes/ai_filters/prompt.rs b/server-rs/src/routes/ai_filters/prompt.rs
index b12fb3c..669e0fa 100644
--- a/server-rs/src/routes/ai_filters/prompt.rs
+++ b/server-rs/src/routes/ai_filters/prompt.rs
@@ -25,9 +25,9 @@ pub fn build_system_prompt(
or \"max\" (at most this value). Never set two filters on the same feature.\n\
- Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
- - \"low crime\" / \"safe\" = low values on the Serious crime (avg/yr) and Minor crime (avg/yr) \
- features (area-normalised incident density near the postcode). Prefer these aggregates for broad \
- area safety; use specific crime features only when the user names a crime type.\n\
+ - \"low crime\" / \"safe\" = low values on the Serious crime (/yr, 7y) and Minor crime (/yr, 7y) \
+ features (average recorded incidents per year near the postcode, last 7 years). Prefer these aggregates for broad \
+ area safety; use specific crime features only when the user names a crime type. Use a \"(/yr, 2y)\" feature only when the user asks about recent crime.\n\
- \"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km \
or low Distance to nearest park (km), depending on wording.\n\
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
@@ -171,8 +171,8 @@ pub fn build_system_prompt(
parts.push(
"\nUser: \"safe quiet area with good schools and parks\"\n\
Output: {\"numeric_filters\": [\
- {\"name\": \"Serious crime (avg/yr)\", \"bound\": \"max\", \"value\": 5}, \
- {\"name\": \"Minor crime (avg/yr)\", \"bound\": \"max\", \"value\": 20}, \
+ {\"name\": \"Serious crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 5}, \
+ {\"name\": \"Minor crime (/yr, 7y)\", \"bound\": \"max\", \"value\": 20}, \
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
{\"name\": \"Good+ primary school catchments\", \"bound\": \"min\", \"value\": 2}, \
{\"name\": \"Good+ secondary school catchments\", \"bound\": \"min\", \"value\": 1}, \
@@ -237,7 +237,7 @@ pub fn build_system_prompt(
"\nUser: \"Labour-voting area with low burglary and a station nearby\"\n\
Output: {\"numeric_filters\": [\
{\"name\": \"% Labour\", \"bound\": \"min\", \"value\": 40}, \
- {\"name\": \"Burglary (avg/yr)\", \"bound\": \"max\", \"value\": 10}, \
+ {\"name\": \"Burglary (/yr, 7y)\", \"bound\": \"max\", \"value\": 10}, \
{\"name\": \"Distance to nearest amenity (Rail station) (km)\", \"bound\": \"max\", \"value\": 1}], \
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
.to_string(),
diff --git a/server-rs/src/routes/crime_records.rs b/server-rs/src/routes/crime_records.rs
new file mode 100644
index 0000000..ff07c7f
--- /dev/null
+++ b/server-rs/src/routes/crime_records.rs
@@ -0,0 +1,194 @@
+//! `GET /api/crime-records` — the individual police.uk crimes (last 7 years)
+//! behind a selected hexagon or postcode, paginated. Display-only and
+//! independent of the property filters, like the population figure: the records
+//! are an attribute of the area, not of the filter-matching subset.
+
+use std::str::FromStr;
+use std::sync::Arc;
+
+use axum::extract::{Query, State};
+use axum::http::StatusCode;
+use axum::response::{IntoResponse, Json};
+use axum::Extension;
+use rustc_hash::{FxHashMap, FxHashSet};
+use serde::{Deserialize, Serialize};
+use tracing::{info, warn};
+
+use crate::auth::OptionalUser;
+use crate::licensing::{check_license_bounds, check_license_point, resolve_share_code};
+use crate::parsing::{cell_for_row_cached, h3_cell_bounds, needs_parent, validate_h3_resolution};
+use crate::state::SharedState;
+use crate::utils::normalize_postcode;
+
+/// Default and hard-cap page sizes for the records list.
+const DEFAULT_LIMIT: usize = 200;
+const MAX_LIMIT: usize = 500;
+
+#[derive(Deserialize)]
+pub struct CrimeRecordsParams {
+ /// Hexagon selection: H3 cell + resolution. Mutually exclusive with `postcode`.
+ pub h3: Option,
+ pub resolution: Option,
+ /// Postcode selection.
+ pub postcode: Option,
+ pub offset: Option,
+ pub limit: Option,
+ /// Lower bound on `month_index` (`year*12 + month0`) to restrict to a recent
+ /// window; omitted = all stored records (last 7 years).
+ pub since: Option,
+ /// Share-link code; grants scoped access for unlicensed users.
+ pub share: Option,
+}
+
+#[derive(Serialize)]
+pub struct CrimeRecord {
+ /// `"YYYY-MM"`.
+ pub month: String,
+ #[serde(rename = "type")]
+ pub crime_type: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub location: Option,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub outcome: Option,
+ pub lat: f32,
+ pub lon: f32,
+}
+
+#[derive(Serialize)]
+pub struct CrimeRecordsResponse {
+ pub records: Vec,
+ pub total: usize,
+ pub offset: usize,
+ pub truncated: bool,
+}
+
+fn format_month(month_index: u32) -> String {
+ let year = month_index / 12;
+ let month = month_index % 12 + 1;
+ format!("{year:04}-{month:02}")
+}
+
+pub async fn get_crime_records(
+ State(shared): State>,
+ Extension(user): Extension,
+ Extension(geo): Extension,
+ Query(params): Query,
+) -> Result, axum::response::Response> {
+ let state = shared.load_state();
+ let share_bounds = resolve_share_code(&state, params.share.as_deref()).await;
+ let offset = params.offset.unwrap_or(0);
+ let limit = params.limit.unwrap_or(DEFAULT_LIMIT).min(MAX_LIMIT);
+ let since = params.since;
+
+ // Resolve the selection to a set of postcodes, after a license check scoped
+ // to the selection's geometry (bounds for a hexagon, point for a postcode).
+ enum Selection {
+ Hexagon { cell: u64, resolution: u8 },
+ Postcode(String),
+ }
+
+ let selection = if let Some(h3) = params.h3.clone() {
+ let cell = h3o::CellIndex::from_str(&h3).map_err(|error| {
+ warn!(h3 = %h3, error = %error, "Invalid H3 cell index");
+ (StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {error}")).into_response()
+ })?;
+ let resolution = params.resolution.ok_or_else(|| {
+ (StatusCode::BAD_REQUEST, "resolution is required with h3").into_response()
+ })?;
+ validate_h3_resolution(resolution).map_err(IntoResponse::into_response)?;
+ let bounds = h3_cell_bounds(cell, 0.0);
+ check_license_bounds(&user.0, bounds, geo.free_zone, share_bounds)?;
+ Selection::Hexagon {
+ cell: cell.into(),
+ resolution,
+ }
+ } else if let Some(postcode) = params.postcode.clone() {
+ let normalized = normalize_postcode(&postcode);
+ let &pc_idx = state
+ .postcode_data
+ .postcode_to_idx
+ .get(&normalized)
+ .ok_or_else(|| {
+ (StatusCode::NOT_FOUND, format!("Postcode not found: {normalized}")).into_response()
+ })?;
+ let (lat, lon) = state.postcode_data.centroids[pc_idx];
+ check_license_point(&user.0, lat as f64, lon as f64, geo.free_zone, share_bounds)?;
+ Selection::Postcode(normalized)
+ } else {
+ return Err((StatusCode::BAD_REQUEST, "h3 or postcode is required").into_response());
+ };
+
+ let result = tokio::task::spawn_blocking(move || -> Result {
+ // Distinct postcodes covered by the selection.
+ let postcodes: Vec = match selection {
+ Selection::Postcode(pc) => vec![pc],
+ Selection::Hexagon { cell, resolution } => {
+ let h3_res = h3o::Resolution::try_from(resolution)
+ .map_err(|err| format!("Invalid H3 resolution {resolution}: {err}"))?;
+ let need_parent = needs_parent(resolution);
+ let h3o_cell = h3o::CellIndex::try_from(cell)
+ .map_err(|err| format!("Invalid H3 cell: {err}"))?;
+ let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(h3o_cell, 0.001);
+ let mut h3_cache: FxHashMap = FxHashMap::default();
+ let mut seen: FxHashSet<&str> = FxHashSet::default();
+ let mut out: Vec = Vec::new();
+ state.grid.for_each_in_bounds(
+ min_lat,
+ min_lon,
+ max_lat,
+ max_lon,
+ |row_idx| {
+ let row = row_idx as usize;
+ if cell_for_row_cached(
+ row,
+ &state.h3_cells,
+ h3_res,
+ need_parent,
+ &mut h3_cache,
+ ) == cell
+ {
+ let pc = state.data.postcode(row);
+ if seen.insert(pc) {
+ out.push(pc.to_string());
+ }
+ }
+ },
+ );
+ out
+ }
+ };
+
+ let pc_refs: Vec<&str> = postcodes.iter().map(String::as_str).collect();
+ let indices = state.crime_records.gather(&pc_refs, since);
+ let total = indices.len();
+ let records: Vec = indices
+ .iter()
+ .skip(offset)
+ .take(limit)
+ .map(|&idx| {
+ let v = state.crime_records.view(idx);
+ CrimeRecord {
+ month: format_month(v.month_index),
+ crime_type: v.crime_type.to_string(),
+ location: v.location.map(str::to_string),
+ outcome: v.outcome.map(str::to_string),
+ lat: v.lat,
+ lon: v.lon,
+ }
+ })
+ .collect();
+ let truncated = offset + records.len() < total;
+ Ok(CrimeRecordsResponse {
+ records,
+ total,
+ offset,
+ truncated,
+ })
+ })
+ .await
+ .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response())?
+ .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error).into_response())?;
+
+ info!(total = result.total, returned = result.records.len(), "GET /api/crime-records");
+ Ok(Json(result))
+}
diff --git a/server-rs/src/routes/hexagon_stats.rs b/server-rs/src/routes/hexagon_stats.rs
index 36c8706..482c980 100644
--- a/server-rs/src/routes/hexagon_stats.rs
+++ b/server-rs/src/routes/hexagon_stats.rs
@@ -74,20 +74,20 @@ pub struct CrimeYearPoint {
#[derive(Serialize)]
pub struct CrimeYearStats {
- /// Underlying crime type (e.g. "Burglary"). Matches existing crime feature
- /// names with the `" (avg/yr)"` suffix stripped.
+ /// Underlying crime type, bare (e.g. "Burglary"). Matches the type prefix of
+ /// the `"(/yr, …)"` crime features.
pub name: String,
pub points: Vec,
}
-/// Average headline crime rate (avg/yr) for one crime type across the
-/// selection's outcode and postcode sector. Comparable to the national average
-/// shown per metric in the right pane.
+/// Average crime count for one crime feature across the selection's outcode and
+/// postcode sector. Comparable to the national average shown per metric in the
+/// right pane.
#[derive(Serialize)]
pub struct CrimeAreaAverage {
- /// Crime type, bare (e.g. "Burglary"). Matches `CrimeYearStats.name`.
+ /// Full crime-feature name (e.g. "Burglary (/yr, 7y)").
pub name: String,
- /// Exact national mean (avg/yr) — the frontend prefers this over the
+ /// Exact national mean count — the frontend prefers this over the
/// histogram-bin national average for crime so all four numbers in the row
/// share one estimator.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -161,10 +161,15 @@ pub struct HexagonStatsResponse {
/// present only when sector crime averages are available for it.
#[serde(skip_serializing_if = "Option::is_none")]
pub crime_sector: Option,
- /// Per-crime-type average rates across the central postcode's outcode and
+ /// Per-crime-type average counts across the central postcode's outcode and
/// sector, shown alongside the national average for each crime metric.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub crime_area_averages: Vec,
+ /// Total individual crime records (last 7 years) across the distinct
+ /// postcodes in this selection — the count behind the "individual crimes"
+ /// list. Filter-independent, like `population`.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub crime_total_records: Option,
#[serde(skip_serializing_if = "Option::is_none")]
pub central_postcode: Option,
/// Total usual residents (ONS Census 2021) living across the distinct
@@ -699,26 +704,42 @@ pub async fn get_hexagon_stats(
&field_set,
);
- // Sum usual residents across the distinct postcodes covered by the
- // hexagon. Computed over `area_rows` (all properties in the cell), not
- // the filter-matching subset, so toggling filters never changes it —
- // population is an attribute of the area, like the council-house count.
+ // Distinct postcodes covered by the hexagon, taken over `area_rows` (all
+ // properties in the cell), not the filter-matching subset — population and
+ // the crime-records count are attributes of the area, independent of the
+ // active filters (like the council-house count).
+ let mut seen: HashSet<&str> = HashSet::new();
+ let mut area_postcodes: Vec<&str> = Vec::new();
+ for &row in &area_rows {
+ let pc = state.data.postcode(row);
+ if seen.insert(pc) {
+ area_postcodes.push(pc);
+ }
+ }
+
let population = {
- let mut seen: HashSet<&str> = HashSet::new();
let mut total: u64 = 0;
let mut found = false;
- for &row in &area_rows {
- let pc = state.data.postcode(row);
- if seen.insert(pc) {
- if let Some(p) = state.population.for_postcode(pc) {
- total += p as u64;
- found = true;
- }
+ for &pc in &area_postcodes {
+ if let Some(p) = state.population.for_postcode(pc) {
+ total += p as u64;
+ found = true;
}
}
found.then(|| total.min(u32::MAX as u64) as u32)
};
+ let crime_total_records = {
+ // Sum the per-postcode counts straight from the CSR index instead of
+ // materializing (and sorting) every record index: this keeps the
+ // mmap-backed columns cold on the hot hexagon path.
+ let total: u64 = area_postcodes
+ .iter()
+ .map(|pc| state.crime_records.total_for(pc) as u64)
+ .sum();
+ (total > 0).then(|| total.min(u32::MAX as u64) as u32)
+ };
+
Ok(HexagonStatsResponse {
count: total_count,
numeric_features,
@@ -729,6 +750,7 @@ pub async fn get_hexagon_stats(
crime_outcode,
crime_sector,
crime_area_averages,
+ crime_total_records,
central_postcode,
population,
filter_exclusions,
diff --git a/server-rs/src/routes/postcode_stats.rs b/server-rs/src/routes/postcode_stats.rs
index a1fe4b7..98219b2 100644
--- a/server-rs/src/routes/postcode_stats.rs
+++ b/server-rs/src/routes/postcode_stats.rs
@@ -227,6 +227,11 @@ pub async fn get_postcode_stats(
// Usual residents (Census 2021) for this postcode. Display-only.
let population = state.population.for_postcode(&postcode_str);
+ let crime_total_records = {
+ let total = state.crime_records.total_for(&postcode_str);
+ (total > 0).then_some(total)
+ };
+
Ok(HexagonStatsResponse {
count: total_count,
numeric_features,
@@ -237,6 +242,7 @@ pub async fn get_postcode_stats(
crime_outcode,
crime_sector,
crime_area_averages,
+ crime_total_records,
central_postcode: None,
population,
filter_exclusions,
diff --git a/server-rs/src/routes/shorten.rs b/server-rs/src/routes/shorten.rs
index b4841a0..0f26cf9 100644
--- a/server-rs/src/routes/shorten.rs
+++ b/server-rs/src/routes/shorten.rs
@@ -127,15 +127,20 @@ fn is_allowed_param_key(key: &str) -> bool {
| "filter"
| "school"
| "crime"
+ | "crimeSeverity"
| "voteShare"
| "ethnicity"
+ | "qualification"
+ | "tenure"
| "amenityDistance"
| "transportDistance"
| "amenityCount2km"
| "amenityCount5km"
| "poi"
| "overlay"
+ | "crimeType"
| "basemap"
+ | "colorOpacity"
| "tab"
| "pc"
| "tt"
@@ -594,6 +599,22 @@ mod tests {
assert_eq!(params, "lat=51.5&lon=-0.1&zoom=12&basemap=satellite");
}
+ #[test]
+ fn preserves_all_filter_params_for_share_links() {
+ // Every filter param emitted by the frontend's stateToParams() must survive
+ // shortening; an unsupported key is rejected outright (see is_allowed_param_key),
+ // which fails the whole share link rather than dropping a single filter.
+ // Values use %3A (":") since form re-serialization keeps it stable.
+ let query = "crimeSeverity=Serious%3A0%3A5\
+ &qualification=Degree%3A20%3A80\
+ &tenure=Owner%3A30%3A90\
+ &crimeType=burglary\
+ &colorOpacity=60";
+ let params = sanitized_query_params(query, false).unwrap();
+
+ assert_eq!(params, query);
+ }
+
#[test]
fn escapes_html_attributes() {
assert_eq!(escape_attr(r#""'><&"#), ""'><&");
diff --git a/server-rs/src/routes/stats.rs b/server-rs/src/routes/stats.rs
index 91577e3..bf1d4cf 100644
--- a/server-rs/src/routes/stats.rs
+++ b/server-rs/src/routes/stats.rs
@@ -11,8 +11,8 @@ use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
use crate::utils::{postcode_outcode, postcode_sector};
use super::hexagon_stats::{
- CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats, HistogramStats,
- NumericFeatureStats, PricePoint,
+ CrimeAreaAverage, CrimeYearPoint, CrimeYearStats, EnumFeatureStats,
+ HistogramStats, NumericFeatureStats, PricePoint,
};
/// Extract price history (year, price) pairs from matching rows, downsampled if needed.
@@ -352,11 +352,14 @@ pub fn compute_crime_by_year(
let mut out = Vec::new();
for (type_idx, name) in crime_by_year.crime_types.iter().enumerate() {
// Crime types in the by-year side table are bare (e.g. "Burglary"), while
- // the configured feature names carry an " (avg/yr)" suffix. Match either
- // form so callers can pass the feature names they already know.
+ // the configured feature names carry a window suffix ("Burglary (/yr,
+ // 7y)"). Emit the bare-type trend if the bare name is requested directly or
+ // any of its windowed features is.
if fields_specified {
- let with_suffix = format!("{name} (avg/yr)");
- if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
+ let prefix = format!("{name} (");
+ if !field_set.contains(name.as_str())
+ && !field_set.iter().any(|f| f.starts_with(&prefix))
+ {
continue;
}
}
@@ -395,6 +398,7 @@ pub fn compute_crime_by_year(
out
}
+
/// Latest year present anywhere in the by-year crime dataset. The client
/// compares each selection's last charted year against this to caption
/// force-level publication gaps (e.g. Greater Manchester ends mid-2019) as
@@ -440,13 +444,10 @@ pub fn area_crime_averages_for(
let mut out = Vec::new();
for (idx, name) in averages.crime_types.iter().enumerate() {
- // Crime types are bare here ("Burglary"); requested fields may carry the
- // " (avg/yr)" suffix — accept either form, like compute_crime_by_year.
- if fields_specified {
- let with_suffix = format!("{name} (avg/yr)");
- if !field_set.contains(name.as_str()) && !field_set.contains(with_suffix.as_str()) {
- continue;
- }
+ // `name` is the full crime-feature name here (e.g. "Burglary (/yr,
+ // 7y)"), matching exactly the feature fields the caller requests.
+ if fields_specified && !field_set.contains(name.as_str()) {
+ continue;
}
let national_val = finite_at(Some(&averages.national), idx);
let outcode_val = finite_at(outcode_means, idx);
@@ -595,7 +596,10 @@ mod tests {
let mut by_sector = rustc_hash::FxHashMap::default();
by_sector.insert("E14 2".to_string(), vec![5.0, 7.0]);
AreaCrimeAverages {
- crime_types: vec!["Burglary".to_string(), "Robbery".to_string()],
+ crime_types: vec![
+ "Burglary (/yr, 7y)".to_string(),
+ "Robbery (/yr, 7y)".to_string(),
+ ],
national: vec![8.0, 6.0],
by_outcode,
by_sector,
@@ -611,12 +615,18 @@ mod tests {
assert_eq!(sector.as_deref(), Some("E14 2"));
assert_eq!(out.len(), 2);
- let burglary = out.iter().find(|c| c.name == "Burglary").unwrap();
+ let burglary = out
+ .iter()
+ .find(|c| c.name == "Burglary (/yr, 7y)")
+ .unwrap();
assert_eq!(burglary.national, Some(8.0));
assert_eq!(burglary.outcode, Some(10.0));
assert_eq!(burglary.sector, Some(5.0));
- let robbery = out.iter().find(|c| c.name == "Robbery").unwrap();
+ let robbery = out
+ .iter()
+ .find(|c| c.name == "Robbery (/yr, 7y)")
+ .unwrap();
assert_eq!(robbery.national, Some(6.0));
// The outcode value was NaN — dropped to None; the sector value is finite.
assert_eq!(robbery.outcode, None);
@@ -626,11 +636,13 @@ mod tests {
#[test]
fn area_crime_averages_respect_fields_filter() {
let avgs = sample_averages();
- // The suffixed feature-name form is accepted, like compute_crime_by_year.
- let fields: HashSet = ["Burglary (avg/yr)".to_string()].into_iter().collect();
+ // Area averages are keyed by the full crime-feature name.
+ let fields: HashSet = ["Burglary (/yr, 7y)".to_string()]
+ .into_iter()
+ .collect();
let (_, _, out) = area_crime_averages_for(Some("E14 2DG"), &avgs, true, &fields);
assert_eq!(out.len(), 1);
- assert_eq!(out[0].name, "Burglary");
+ assert_eq!(out[0].name, "Burglary (/yr, 7y)");
}
#[test]
diff --git a/server-rs/src/state.rs b/server-rs/src/state.rs
index 864a9c7..948af4b 100644
--- a/server-rs/src/state.rs
+++ b/server-rs/src/state.rs
@@ -6,9 +6,9 @@ use rustc_hash::FxHashMap;
use crate::auth::TokenCache;
use crate::bugsink::FrontendConfig as BugsinkFrontendConfig;
use crate::data::{
- ActualListingData, AreaCrimeAverages, CrimeByYearData, DevelopmentData, OutcodeData,
- POICategoryGroup, POIData, PlaceData, PostcodeData, PostcodePopulation, PropertyData,
- TravelTimeStore,
+ ActualListingData, AreaCrimeAverages, CrimeByYearData, CrimeRecords,
+ DevelopmentData, OutcodeData, POICategoryGroup, POIData, PlaceData, PostcodeData,
+ PostcodePopulation, PropertyData, TravelTimeStore,
};
use crate::licensing::ShareBoundsCache;
use crate::pocketbase::SuperuserTokenCache;
@@ -52,10 +52,13 @@ pub struct AppState {
pub developments: Arc,
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
pub crime_by_year: Arc,
+ /// Per-postcode individual crime records (last 7 years), spill-backed,
+ /// served by the `/api/crime-records` endpoint and counted in stats.
+ pub crime_records: Arc,
/// Per-unit-postcode usual-resident headcounts (Census 2021), shown in the
/// right pane. Display-only — never filterable. Empty when no data is loaded.
pub population: Arc,
- /// Precomputed per-outcode and per-postcode-sector average crime rates,
+ /// Precomputed per-outcode and per-postcode-sector average crime counts,
/// shown in the right pane alongside the national average for each metric.
pub area_crime_averages: Arc,
/// Token validation cache (60s TTL)
@@ -178,6 +181,7 @@ impl AppState {
series_by_postcode: FxHashMap::default(),
covered_years_by_postcode: FxHashMap::default(),
}),
+ crime_records: Arc::new(CrimeRecords::empty()),
population: Arc::new(PostcodePopulation::empty()),
area_crime_averages: Arc::new(AreaCrimeAverages::empty()),
token_cache: Arc::new(TokenCache::new()),