This commit is contained in:
Andras Schmelczer 2026-07-03 18:39:34 +01:00
parent 1ee796b282
commit ab688243d7
36 changed files with 307 additions and 135 deletions

View file

@ -20,6 +20,7 @@ import { BookmarkIcon } from '../ui/icons/BookmarkIcon';
import { TrashIcon } from '../ui/icons/TrashIcon'; import { TrashIcon } from '../ui/icons/TrashIcon';
import { CloseIcon } from '../ui/icons/CloseIcon'; import { CloseIcon } from '../ui/icons/CloseIcon';
import { useLicense } from '../../hooks/useLicense'; import { useLicense } from '../../hooks/useLicense';
import { useModalA11y } from '../../hooks/useModalA11y';
function PageLayout({ children }: { children: React.ReactNode }) { function PageLayout({ children }: { children: React.ReactNode }) {
return ( return (
@ -43,17 +44,43 @@ function DeleteDialog({
onConfirm: () => void; onConfirm: () => void;
}) { }) {
const { t } = useTranslation(); const { t } = useTranslation();
const dialogRef = useModalA11y();
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'Escape') onCancel();
};
document.addEventListener('keydown', handleKeyDown);
return () => document.removeEventListener('keydown', handleKeyDown);
}, [onCancel]);
return ( return (
<div className="fixed inset-0 z-50 flex items-center justify-center" onClick={onCancel}>
<div className="absolute inset-0 bg-black/50 dark:bg-black/70" />
<div <div
className="relative w-full max-w-sm mx-4 bg-white dark:bg-warm-800 rounded-lg shadow-xl border border-warm-200 dark:border-warm-700" className="fixed inset-0 z-50 flex items-center justify-center"
onClick={onCancel}
role="presentation"
>
<div className="absolute inset-0 bg-black/50 dark:bg-black/70" aria-hidden="true" />
<div
ref={dialogRef}
role="dialog"
aria-modal="true"
aria-labelledby="delete-search-dialog-title"
tabIndex={-1}
className="relative w-full max-w-sm mx-4 bg-white dark:bg-warm-800 rounded-lg shadow-xl border border-warm-200 dark:border-warm-700 outline-none"
onClick={(e) => e.stopPropagation()} onClick={(e) => e.stopPropagation()}
> >
<div className="flex items-center justify-between px-5 pt-5 pb-3"> <div className="flex items-center justify-between px-5 pt-5 pb-3">
<h2 className="text-lg font-semibold text-navy-950 dark:text-white">{title}</h2> <h2
id="delete-search-dialog-title"
className="text-lg font-semibold text-navy-950 dark:text-white"
>
{title}
</h2>
<button <button
type="button"
onClick={onCancel} onClick={onCancel}
aria-label={t('common.close')}
className="text-warm-400 hover:text-warm-700 dark:text-warm-400 dark:hover:text-warm-200" className="text-warm-400 hover:text-warm-700 dark:text-warm-400 dark:hover:text-warm-200"
> >
<CloseIcon className="w-5 h-5" /> <CloseIcon className="w-5 h-5" />
@ -62,12 +89,14 @@ function DeleteDialog({
<p className="px-5 pb-4 text-sm text-warm-700 dark:text-warm-300">{message}</p> <p className="px-5 pb-4 text-sm text-warm-700 dark:text-warm-300">{message}</p>
<div className="flex gap-3 justify-end px-5 pb-5"> <div className="flex gap-3 justify-end px-5 pb-5">
<button <button
type="button"
onClick={onCancel} onClick={onCancel}
className="px-4 py-2 text-sm rounded border border-warm-200 dark:border-warm-700 text-warm-700 dark:text-warm-300 hover:bg-warm-50 dark:hover:bg-warm-700" className="px-4 py-2 text-sm rounded border border-warm-200 dark:border-warm-700 text-warm-700 dark:text-warm-300 hover:bg-warm-50 dark:hover:bg-warm-700"
> >
{t('common.cancel')} {t('common.cancel')}
</button> </button>
<button <button
type="button"
onClick={onConfirm} onClick={onConfirm}
className="px-4 py-2 text-sm rounded bg-red-600 text-white font-medium hover:bg-red-700" className="px-4 py-2 text-sm rounded bg-red-600 text-white font-medium hover:bg-red-700"
> >
@ -124,6 +153,7 @@ function NotesInput({ value, onSave }: { value: string; onSave: (notes: string)
value={text} value={text}
onChange={handleChange} onChange={handleChange}
onBlur={handleBlur} onBlur={handleBlur}
aria-label={t('savedPage.notesPlaceholder')}
placeholder={t('savedPage.notesPlaceholder')} placeholder={t('savedPage.notesPlaceholder')}
rows={1} rows={1}
className="w-full resize-none overflow-hidden rounded border border-warm-200 dark:border-warm-700 bg-warm-50 dark:bg-warm-900 px-3 py-1.5 text-sm text-warm-700 dark:text-warm-300 placeholder-warm-400 dark:placeholder-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400" className="w-full resize-none overflow-hidden rounded border border-warm-200 dark:border-warm-700 bg-warm-50 dark:bg-warm-900 px-3 py-1.5 text-sm text-warm-700 dark:text-warm-300 placeholder-warm-400 dark:placeholder-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400"
@ -160,6 +190,7 @@ function EditableName({ value, onSave }: { value: string; onSave: (name: string)
<input <input
ref={inputRef} ref={inputRef}
value={text} value={text}
aria-label={t('savedPage.clickToRename')}
onChange={(e) => setText(e.target.value)} onChange={(e) => setText(e.target.value)}
onKeyDown={(e) => { onKeyDown={(e) => {
if (e.key === 'Enter') commit(); if (e.key === 'Enter') commit();
@ -175,12 +206,15 @@ function EditableName({ value, onSave }: { value: string; onSave: (name: string)
} }
return ( return (
<h3 <h3 className="font-medium text-navy-950 dark:text-warm-100 truncate">
<button
type="button"
onClick={() => setEditing(true)} onClick={() => setEditing(true)}
className="font-medium text-navy-950 dark:text-warm-100 truncate cursor-pointer hover:text-teal-600 dark:hover:text-teal-400 border-b border-dotted border-transparent hover:border-warm-400 dark:hover:border-warm-500"
title={t('savedPage.clickToRename')} title={t('savedPage.clickToRename')}
className="w-full truncate text-left cursor-pointer hover:text-teal-600 dark:hover:text-teal-400 border-b border-dotted border-transparent hover:border-warm-400 dark:hover:border-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400 rounded"
> >
{value} {value}
</button>
</h3> </h3>
); );
} }
@ -704,7 +738,7 @@ function InviteSection({ user }: { user: AuthUser }) {
return next; return next;
}); });
} catch { } catch {
// Silent non-critical // Silent, non-critical
} finally { } finally {
setInviteHistoryLoading(false); setInviteHistoryLoading(false);
} }
@ -853,6 +887,14 @@ export default function AccountPage({
const { t } = useTranslation(); const { t } = useTranslation();
const [newsletterSaving, setNewsletterSaving] = useState(false); const [newsletterSaving, setNewsletterSaving] = useState(false);
const [newsletterError, setNewsletterError] = useState<string | null>(null); const [newsletterError, setNewsletterError] = useState<string | null>(null);
// Mirror the loaded account record's value locally so the checkbox always
// reflects the stored value and updates instantly on toggle, instead of the
// possibly-stale global auth-store value the async refresh hasn't reconciled
// yet (the source of the "checked while stored value is false" flicker).
const [newsletterChecked, setNewsletterChecked] = useState(user.newsletter);
useEffect(() => {
setNewsletterChecked(user.newsletter);
}, [user.newsletter]);
const { startCheckout, checkingOut, error: checkoutError } = useLicense(); const { startCheckout, checkingOut, error: checkoutError } = useLicense();
const isLicensed = user.subscription === 'licensed' || user.isAdmin; const isLicensed = user.subscription === 'licensed' || user.isAdmin;
@ -916,10 +958,11 @@ export default function AccountPage({
<label className="flex items-center gap-3 cursor-pointer"> <label className="flex items-center gap-3 cursor-pointer">
<input <input
type="checkbox" type="checkbox"
checked={user.newsletter} checked={newsletterChecked}
disabled={newsletterSaving} disabled={newsletterSaving}
onChange={async (e) => { onChange={async (e) => {
const checked = e.target.checked; const checked = e.target.checked;
setNewsletterChecked(checked); // optimistic; reconciled by onRefreshAuth
setNewsletterSaving(true); setNewsletterSaving(true);
setNewsletterError(null); setNewsletterError(null);
try { try {
@ -933,6 +976,7 @@ export default function AccountPage({
assertOk(res, 'Update newsletter'); assertOk(res, 'Update newsletter');
await onRefreshAuth(); await onRefreshAuth();
} catch (err) { } catch (err) {
setNewsletterChecked(!checked); // revert the optimistic toggle on failure
const msg = const msg =
err instanceof Error ? err.message : t('accountPage.updateNewsletterError'); err instanceof Error ? err.message : t('accountPage.updateNewsletterError');
setNewsletterError(msg); setNewsletterError(msg);

View file

@ -0,0 +1,63 @@
// @vitest-environment jsdom
import { cleanup, render, screen } from '@testing-library/react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import InvitePage from './InvitePage';
vi.mock('react-i18next', async (importOriginal) => ({
...(await importOriginal<typeof import('react-i18next')>()),
useTranslation: () => ({ t: (key: string) => key, i18n: { language: 'en' } }),
}));
function mockFetch(invite: { ok: boolean; status: number; body?: unknown }) {
return vi.fn((input: RequestInfo | URL) => {
const url = String(input);
if (url.includes('/api/invite/')) {
return Promise.resolve({
ok: invite.ok,
status: invite.status,
json: () => Promise.resolve(invite.body ?? {}),
} as Response);
}
return Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve({ current_price_pence: 4999 }),
} as Response);
});
}
const baseProps = {
user: null,
theme: 'light' as const,
onLoginClick: vi.fn(),
onRegisterClick: vi.fn(),
onLicenseGranted: vi.fn(),
};
afterEach(() => {
cleanup();
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
describe('InvitePage invalid-invite messaging', () => {
it('shows the unified copy for an unknown code (200 valid:false)', async () => {
vi.stubGlobal(
'fetch',
mockFetch({
ok: true,
status: 200,
body: { valid: false, invite_type: '', used: false, invited_by: null },
})
);
render(<InvitePage code="BOGUS" {...baseProps} />);
expect(await screen.findByText('invitePage.invalidInviteLink')).toBeTruthy();
});
it('shows the SAME copy for a malformed code rejected with 400', async () => {
vi.stubGlobal('fetch', mockFetch({ ok: false, status: 400 }));
render(<InvitePage code="%20" {...baseProps} />);
expect(await screen.findByText('invitePage.invalidInviteLink')).toBeTruthy();
expect(screen.queryByText('invitePage.invalidInvite')).toBeNull();
});
});

View file

@ -115,7 +115,20 @@ export default function InvitePage({
fetch(apiUrl(`invite/${encodeURIComponent(code)}`)), fetch(apiUrl(`invite/${encodeURIComponent(code)}`)),
fetch(apiUrl('pricing')), fetch(apiUrl('pricing')),
]); ]);
if (!inviteRes.ok) throw new Error('Failed to validate invite'); if (!inviteRes.ok) {
// A 4xx means the code itself is malformed or unknown, e.g. /invite/%20
// is rejected with 400 while /invite/BOGUS returns 200 {valid:false}.
// Render the SAME 'invalid invite link' copy for both so messaging is
// consistent; only 5xx / network errors fall through to the transient
// error state below.
if (inviteRes.status >= 400 && inviteRes.status < 500) {
if (!cancelled) {
setInvite({ valid: false, invite_type: '', used: false, invited_by: null });
}
return;
}
throw new Error('Failed to validate invite');
}
const data: InviteInfo = await inviteRes.json(); const data: InviteInfo = await inviteRes.json();
if (!cancelled) setInvite(data); if (!cancelled) setInvite(data);
if (pricingRes.ok) { if (pricingRes.ok) {
@ -230,7 +243,9 @@ export default function InvitePage({
<div className="flex-1 flex items-center justify-center bg-gradient-to-b from-navy-950 via-navy-900 to-navy-900 relative overflow-hidden"> <div className="flex-1 flex items-center justify-center bg-gradient-to-b from-navy-950 via-navy-900 to-navy-900 relative overflow-hidden">
<HexCanvas isDark={isDark} /> <HexCanvas isDark={isDark} />
<div className="text-center relative z-10"> <div className="text-center relative z-10">
<p className="text-lg font-medium text-white mb-2">{t('invitePage.invalidInvite')}</p> <p className="text-lg font-medium text-white mb-2">
{t('invitePage.couldNotValidateTitle')}
</p>
<p className="text-warm-400">{error}</p> <p className="text-warm-400">{error}</p>
</div> </div>
</div> </div>

View file

@ -13,7 +13,6 @@ import {
} from '../../lib/seoLandingPages'; } from '../../lib/seoLandingPages';
import { safeJsonLd } from '../../lib/json-ld'; import { safeJsonLd } from '../../lib/json-ld';
const PUBLIC_URL = 'https://perfect-postcode.co.uk';
const ProductShowcase = lazy(() => import('../home/ProductShowcase')); const ProductShowcase = lazy(() => import('../home/ProductShowcase'));
function ProductShowcaseFallback() { function ProductShowcaseFallback() {
@ -102,32 +101,11 @@ export default function SeoContentPage({
}) { }) {
const { t, i18n } = useTranslation(); const { t, i18n } = useTranslation();
const page = getLocalizedSeoContentPage(pageKey, i18n.language); const page = getLocalizedSeoContentPage(pageKey, i18n.language);
const url = `${PUBLIC_URL}${page.path}`;
usePageMeta(page.metaTitle, page.metaDescription); usePageMeta(page.metaTitle, page.metaDescription);
return ( return (
<main className="flex-1 overflow-y-auto bg-warm-50 text-navy-950 dark:bg-navy-950 dark:text-warm-100"> <main className="flex-1 overflow-y-auto bg-warm-50 text-navy-950 dark:bg-navy-950 dark:text-warm-100">
<FaqJsonLd faq={page.faq} /> <FaqJsonLd faq={page.faq} />
<JsonLd
data={{
'@context': 'https://schema.org',
'@type': 'BreadcrumbList',
itemListElement: [
{
'@type': 'ListItem',
position: 1,
name: 'Perfect Postcode',
item: `${PUBLIC_URL}/`,
},
{
'@type': 'ListItem',
position: 2,
name: page.title,
item: url,
},
],
}}
/>
<section className="bg-navy-950 text-white"> <section className="bg-navy-950 text-white">
<div className="mx-auto max-w-5xl px-6 py-16 md:px-10 md:py-20"> <div className="mx-auto max-w-5xl px-6 py-16 md:px-10 md:py-20">

View file

@ -10,6 +10,7 @@ import { getElectionVoteShareFeatureName } from '../../lib/election-filter';
import { getEthnicityFeatureName } from '../../lib/ethnicity-filter'; import { getEthnicityFeatureName } from '../../lib/ethnicity-filter';
import { getQualificationFeatureName } from '../../lib/qualification-filter'; import { getQualificationFeatureName } from '../../lib/qualification-filter';
import { getTenureFeatureName } from '../../lib/tenure-filter'; import { getTenureFeatureName } from '../../lib/tenure-filter';
import { getCouncilFeatureName } from '../../lib/council-filter';
import { import {
POI_DISTANCE_FILTER_NAME, POI_DISTANCE_FILTER_NAME,
getPoiDistanceFeatureName, getPoiDistanceFeatureName,
@ -60,6 +61,7 @@ export default memo(function HoverCard({
const ethnicityFeatureName = getEthnicityFeatureName(name); const ethnicityFeatureName = getEthnicityFeatureName(name);
const qualificationFeatureName = getQualificationFeatureName(name); const qualificationFeatureName = getQualificationFeatureName(name);
const tenureFeatureName = getTenureFeatureName(name); const tenureFeatureName = getTenureFeatureName(name);
const councilFeatureName = getCouncilFeatureName(name);
const poiDistanceFeatureName = getPoiDistanceFeatureName(name); const poiDistanceFeatureName = getPoiDistanceFeatureName(name);
const backendName = const backendName =
schoolBackendName ?? schoolBackendName ??
@ -69,6 +71,7 @@ export default memo(function HoverCard({
ethnicityFeatureName ?? ethnicityFeatureName ??
qualificationFeatureName ?? qualificationFeatureName ??
tenureFeatureName ?? tenureFeatureName ??
councilFeatureName ??
poiDistanceFeatureName ?? poiDistanceFeatureName ??
name; name;
const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`]; const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`];

View file

@ -231,7 +231,7 @@ export default function LocationSearch({
return; return;
} }
// Postcode fetch geometry. Unlike place/address results, a postcode // Postcode: fetch geometry. Unlike place/address results, a postcode
// result carries no coordinates, so the camera move genuinely depends on // result carries no coordinates, so the camera move genuinely depends on
// this response and stays gated by isCurrentLookup. // this response and stays gated by isCurrentLookup.
try { try {

View file

@ -9,11 +9,11 @@ import { MapFallback } from './map-page/Fallbacks';
* *
* deck.gl runs in interleaved mode, sharing MapLibre's WebGL context. When that * deck.gl runs in interleaved mode, sharing MapLibre's WebGL context. When that
* shared context gets into a bad state (lost context, a buffer/texture finalized * shared context gets into a bad state (lost context, a buffer/texture finalized
* while still referenced the "bufferSubData: no buffer" / "bindTexture: deleted * while still referenced, the "bufferSubData: no buffer" / "bindTexture: deleted
* object" storm), the next interleaved draw can throw. deck.gl's own animation * object" storm), the next interleaved draw can throw. deck.gl's own animation
* loop swallows render errors via its `onError` prop, but interleaved draws fire * loop swallows render errors via its `onError` prop, but interleaved draws fire
* from MapLibre's synchronous `render` event, which can run inside a React commit * from MapLibre's synchronous `render` event, which can run inside a React commit,
* so the throw bubbles to the nearest React error boundary. * so the throw bubbles to the nearest React error boundary.
* *
* Without this, that throw reaches the single top-level boundary and blanks the * Without this, that throw reaches the single top-level boundary and blanks the
* entire app ("Something went wrong / Refresh the page"). This boundary contains * entire app ("Something went wrong / Refresh the page"). This boundary contains
@ -26,7 +26,7 @@ const MAX_AUTO_RECOVERIES = 2;
// Remount after a short delay so a transient GL state can settle first. // Remount after a short delay so a transient GL state can settle first.
const RECOVERY_DELAY_MS = 400; const RECOVERY_DELAY_MS = 400;
// Crashes spaced further apart than this are treated as independent, so the // Crashes spaced further apart than this are treated as independent, so the
// auto-recovery budget resets — only a tight crash loop escalates to manual retry. // auto-recovery budget resets. Only a tight crash loop escalates to manual retry.
const STABILITY_WINDOW_MS = 30_000; const STABILITY_WINDOW_MS = 30_000;
interface MapErrorBoundaryProps { interface MapErrorBoundaryProps {

View file

@ -113,7 +113,7 @@ export default function MobileDrawer({
> >
<div className="h-[10%] shrink-0" aria-hidden="true" /> <div className="h-[10%] shrink-0" aria-hidden="true" />
{/* Panel bottom 90% */} {/* Panel: bottom 90% */}
<div <div
ref={panelRef} ref={panelRef}
className="pointer-events-auto h-[90%] bg-white dark:bg-navy-950 rounded-t-2xl flex flex-col border-t border-x border-warm-300 ring-1 ring-navy-950/10 shadow-2xl shadow-navy-950/45 dark:border-navy-600 dark:ring-white/10 dark:shadow-black/60 overflow-hidden" className="pointer-events-auto h-[90%] bg-white dark:bg-navy-950 rounded-t-2xl flex flex-col border-t border-x border-warm-300 ring-1 ring-navy-950/10 shadow-2xl shadow-navy-950/45 dark:border-navy-600 dark:ring-white/10 dark:shadow-black/60 overflow-hidden"

View file

@ -40,8 +40,8 @@ describe('computeNumberLineLayout', () => {
300, 300,
fmt fmt
)!; )!;
const low = layout.items.find((i) => i.kind === 'national')!; // 30 lowest const low = layout.items.find((i) => i.kind === 'national')!; // 30: lowest
const high = layout.items.find((i) => i.kind === 'area')!; // 50 highest const high = layout.items.find((i) => i.kind === 'area')!; // 50: highest
expect(low.tickX).toBeCloseTo(layout.plotLeft, 5); expect(low.tickX).toBeCloseTo(layout.plotLeft, 5);
expect(high.tickX).toBeCloseTo(layout.plotRight, 5); expect(high.tickX).toBeCloseTo(layout.plotRight, 5);
}); });

View file

@ -120,7 +120,7 @@ export function computeNumberLineLayout(
* spanning the lowesthighest value, so the selection can be read against its * spanning the lowesthighest value, so the selection can be read against its
* reference points (national / outcode / sector) at a glance. Labels are spread * reference points (national / outcode / sector) at a glance. Labels are spread
* to avoid overlap * to avoid overlap
* common since the nested area/sector/outcode values cluster and connected * (common since the nested area/sector/outcode values cluster) and connected
* back to their tick with a thin leader line. * back to their tick with a thin leader line.
*/ */
export default function NumberLine({ points, format }: NumberLineProps) { export default function NumberLine({ points, format }: NumberLineProps) {

View file

@ -11,6 +11,7 @@ import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter'; import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
import { getQualificationFeatureName } from '../../../lib/qualification-filter'; import { getQualificationFeatureName } from '../../../lib/qualification-filter';
import { getTenureFeatureName } from '../../../lib/tenure-filter'; import { getTenureFeatureName } from '../../../lib/tenure-filter';
import { getCouncilFeatureName } from '../../../lib/council-filter';
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter'; import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
import { getSchoolBackendFeatureName } from '../../../lib/school-filter'; import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
@ -31,6 +32,7 @@ export function getMapPageBackendFeatureName(featureName: string): string {
getEthnicityFeatureName(featureName) ?? getEthnicityFeatureName(featureName) ??
getQualificationFeatureName(featureName) ?? getQualificationFeatureName(featureName) ??
getTenureFeatureName(featureName) ?? getTenureFeatureName(featureName) ??
getCouncilFeatureName(featureName) ??
getPoiDistanceFeatureName(featureName) ?? getPoiDistanceFeatureName(featureName) ??
featureName featureName
); );

View file

@ -4,6 +4,7 @@ export const Map = lazy(() => import('../Map'));
export const Filters = lazy(() => import('../Filters')); export const Filters = lazy(() => import('../Filters'));
export const POIPane = lazy(() => import('../POIPane')); export const POIPane = lazy(() => import('../POIPane'));
export const OverlayPane = lazy(() => import('../OverlayPane')); export const OverlayPane = lazy(() => import('../OverlayPane'));
export const ListingPane = lazy(() => import('../ListingPane'));
export const AreaPane = lazy(() => import('../AreaPane')); export const AreaPane = lazy(() => import('../AreaPane'));
export const PropertiesPane = lazy(() => export const PropertiesPane = lazy(() =>
import('../PropertiesPane').then((module) => ({ default: module.PropertiesPane })) import('../PropertiesPane').then((module) => ({ default: module.PropertiesPane }))

View file

@ -107,7 +107,7 @@ def match_schools(truth: pl.DataFrame, gias: pl.DataFrame) -> pl.DataFrame:
.alias("_pc"), .alias("_pc"),
).with_row_index("_row_id") ).with_row_index("_row_id")
# 1. Exact postcode match (unique postcodes only site-sharing schools # 1. Exact postcode match (unique postcodes only: site-sharing schools
# would mismatch phases otherwise; those fall through to name matching). # would mismatch phases otherwise; those fall through to name matching).
pc_unique = gias.filter(pl.col("_pc").is_not_null()).unique( pc_unique = gias.filter(pl.col("_pc").is_not_null()).unique(
subset="_pc", keep="none" subset="_pc", keep="none"
@ -291,7 +291,7 @@ def main() -> None:
print(f"\nWrote matched rows to {args.matched_out}") print(f"\nWrote matched rows to {args.matched_out}")
if binding.is_empty(): if binding.is_empty():
raise SystemExit("No binding, matchable cutoffs nothing to calibrate on") raise SystemExit("No binding, matchable cutoffs: nothing to calibrate on")
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -5,11 +5,11 @@ computation failed or was interrupted, leaving either zero rows or only
the origin postcode. We detect this by an absolute, structural criterion: the origin postcode. We detect this by an absolute, structural criterion:
a file is corrupt only when it is unreadable or has a row count at or below a file is corrupt only when it is unreadable or has a row count at or below
CORRUPT_ROW_FLOOR. Per-mode percentile/median/range figures are reported CORRUPT_ROW_FLOOR. Per-mode percentile/median/range figures are reported
for context only they never drive the deletable set, so repeated runs for context only. They never drive the deletable set, so repeated runs
(including with --delete) are idempotent and never erode legitimate (including with --delete) are idempotent and never erode legitimate
small-catchment (rural/island) origins. small-catchment (rural/island) origins.
Duplicates arise when places.parquet is rebuilt between R5 runs each Duplicates arise when places.parquet is rebuilt between R5 runs. Each
place gets a new numeric index prefix, so the skip-completed logic place gets a new numeric index prefix, so the skip-completed logic
doesn't recognize previous results. --dedup keeps only the largest doesn't recognize previous results. --dedup keeps only the largest
file per slug and removes the rest. file per slug and removes the rest.
@ -105,7 +105,7 @@ def find_bad_files(base_dir: Path) -> tuple[list[BadFile], dict[str, dict]]:
if not row_counts: if not row_counts:
continue continue
# Reporting statistics only these never decide what gets deleted. # Reporting statistics only; these never decide what gets deleted.
p5 = percentile(row_counts, 5) p5 = percentile(row_counts, 5)
median = percentile(row_counts, 50) median = percentile(row_counts, 50)

View file

@ -16,7 +16,7 @@ PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/resea
# Pre-staged file path. Ofcom put the entire ofcom.org.uk domain behind # Pre-staged file path. Ofcom put the entire ofcom.org.uk domain behind
# Cloudflare's Managed Challenge in 2026, which requires a JS-executing # Cloudflare's Managed Challenge in 2026, which requires a JS-executing
# browser to pass — no amount of User-Agent / TLS-impersonation spoofing # browser to pass. No amount of User-Agent / TLS-impersonation spoofing
# (curl_cffi chrome120..131, safari17, firefox133, chrome_android) gets # (curl_cffi chrome120..131, safari17, firefox133, chrome_android) gets
# past it. When the automated download fails, the user must download the # past it. When the automated download fails, the user must download the
# zip manually from the Source URL above and place it at this path. # zip manually from the Source URL above and place it at this path.

View file

@ -63,7 +63,7 @@ def _fetch_csv(
"""Download a CSV, defending against silent truncation. """Download a CSV, defending against silent truncation.
The NOMIS bulk files are served with chunked transfer encoding and no The NOMIS bulk files are served with chunked transfer encoding and no
Content-Length, so a dropped connection ends the stream without raising Content-Length, so a dropped connection ends the stream without raising,
yielding a short file. We retry until the parsed row count reaches the yielding a short file. We retry until the parsed row count reaches the
file's known approximate size, keeping the largest parse seen. file's known approximate size, keeping the largest parse seen.
""" """
@ -72,7 +72,7 @@ def _fetch_csv(
try: try:
download(url, dest) download(url, dest)
df = pl.read_csv(dest) df = pl.read_csv(dest)
except Exception as exc: # noqa: BLE001 retry any transport/parse error except Exception as exc: # noqa: BLE001: retry any transport/parse error
print(f" {url} attempt {attempt}: error {exc}") print(f" {url} attempt {attempt}: error {exc}")
time.sleep(3) time.sleep(3)
continue continue

View file

@ -193,7 +193,7 @@ def select_coverage_archives(
adjacent one) matters when the adjacent snapshot is missing from the index: adjacent one) matters when the adjacent snapshot is missing from the index:
skipping it would leave a multi-month hole, while overlap only costs skipping it would leave a multi-month hole, while overlap only costs
download time because extraction skips already-extracted months. A hole no download time because extraction skips already-extracted months. A hole no
archive can bridge is a publication gap in the source a hard error unless archive can bridge is a publication gap in the source: a hard error unless
``allow_gaps``, since the run would otherwise be stamped complete with ``allow_gaps``, since the run would otherwise be stamped complete with
artificial dips in every crime-over-time series. artificial dips in every crime-over-time series.
""" """

View file

@ -143,8 +143,8 @@ def _has_dwellings(min_d: int | None, max_d: int | None) -> bool:
def _brownfield_url(entity) -> str | None: def _brownfield_url(entity) -> str | None:
"""Canonical per-site page on the Planning Data platform. """Canonical per-site page on the Planning Data platform.
The register's own ``site-plan-url`` is unreliable many LPAs point every The register's own ``site-plan-url`` is unreliable (many LPAs point every
row at a single generic register landing page so we link to the stable, row at a single generic register landing page), so we link to the stable,
always-per-site entity page instead. always-per-site entity page instead.
""" """
entity_id = _to_int(entity) entity_id = _to_int(entity)

View file

@ -10,13 +10,13 @@ rather than a single headline number.
We give the ONS bands colloquial labels (the census's "Level 1/2/3/4+" jargon We give the ONS bands colloquial labels (the census's "Level 1/2/3/4+" jargon
means little to a homebuyer): No qualifications / Some GCSEs / Good GCSEs / means little to a homebuyer): No qualifications / Some GCSEs / Good GCSEs /
Apprenticeship / A-levels / Degree or higher / Other qualifications. NOTE the Apprenticeship / A-levels / Degree or higher / Other qualifications. NOTE the
census does NOT split undergraduate from postgraduate "Level 4 and above" is a census does NOT split undergraduate from postgraduate. "Level 4 and above" is a
single bucket ("Degree or higher"). single bucket ("Degree or higher").
The join key downstream (merge.py) is `lsoa21`, the same key used for ethnicity, The join key downstream (merge.py) is `lsoa21`, the same key used for ethnicity,
median age, and IoD. median age, and IoD.
Source: NOMIS (ONS Census 2021 TS067 dataset, NM_2084_1) Source: NOMIS (ONS Census 2021, TS067 dataset, NM_2084_1)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
""" """
@ -42,7 +42,7 @@ BASE_URL = (
) )
# Map each canonical NOMIS C2021_HIQUAL_8 band to our colloquial output bucket. # Map each canonical NOMIS C2021_HIQUAL_8 band to our colloquial output bucket.
# 1:1 mapping (no folding) keyed on the exact NOMIS label so a relabelled or # 1:1 mapping (no folding): keyed on the exact NOMIS label so a relabelled or
# missing band fails loudly in validation instead of silently dropping people. # missing band fails loudly in validation instead of silently dropping people.
BAND_MAP = { BAND_MAP = {
"No qualifications": "No qualifications", "No qualifications": "No qualifications",

View file

@ -5,7 +5,7 @@ import httpx
import polars as pl import polars as pl
# UK Parliament publishes candidate-level results for the 2024 General Election. # UK Parliament publishes candidate-level results for the 2024 General Election.
# One row per candidate per constituency — we aggregate to per-constituency stats. # One row per candidate per constituency. We aggregate to per-constituency stats.
URL = "https://electionresults.parliament.uk/general-elections/6/candidacies.csv" URL = "https://electionresults.parliament.uk/general-elections/6/candidacies.csv"
# Map party names to a smaller set for vote share columns. # Map party names to a smaller set for vote share columns.

View file

@ -9,7 +9,7 @@ from pathlib import Path
import httpx import httpx
# ArcGIS REST API query for England only, generalised (BGC) resolution # ArcGIS REST API: query for England only, generalised (BGC) resolution
URL = ( URL = (
"https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/" "https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/"
"Countries_December_2024_Boundaries_UK_BGC/FeatureServer/0/query" "Countries_December_2024_Boundaries_UK_BGC/FeatureServer/0/query"

View file

@ -11,7 +11,7 @@ different neighbourhoods in one borough no longer share an identical ethnicity
profile. The join key downstream (merge.py) is `lsoa21`, the same key already profile. The join key downstream (merge.py) is `lsoa21`, the same key already
used for median age and IoD. used for median age and IoD.
Source: NOMIS (ONS Census 2021 TS021 dataset, NM_2041_1) Source: NOMIS (ONS Census 2021, TS021 dataset, NM_2041_1)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
""" """

View file

@ -4,12 +4,12 @@ GIAS is the DfE register of all educational establishments in England, updated
daily. The CSV is generated on-demand via a four-step interaction with the daily. The CSV is generated on-demand via a four-step interaction with the
public Downloads page (there is no static URL): public Downloads page (there is no static URL):
1. GET /Downloads extract anti-forgery token, the `all.edubase.data` tag, 1. GET /Downloads: extract anti-forgery token, the `all.edubase.data` tag,
and the FileGeneratedDate that the server expects for that tag today. and the FileGeneratedDate that the server expects for that tag today.
2. POST /Downloads/Collate submit the form to start file generation. The 2. POST /Downloads/Collate: submit the form to start file generation. The
redirect URL contains a generation UUID. redirect URL contains a generation UUID.
3. Poll /Downloads/GenerateAjax/{id} until status:true. 3. Poll /Downloads/GenerateAjax/{id} until status:true.
4. GET the Azure blob URL with ?id={id} returns a ZIP containing 4. GET the Azure blob URL with ?id={id}: returns a ZIP containing
`edubasealldataYYYYMMDD.csv`. `edubasealldataYYYYMMDD.csv`.
The CSV is cp1252-encoded with 135 columns. We keep the fields useful for a The CSV is cp1252-encoded with 135 columns. We keep the fields useful for a
@ -262,7 +262,7 @@ def transform(zip_bytes: bytes) -> pl.DataFrame:
.cast(pl.Float32, strict=False), .cast(pl.Float32, strict=False),
) )
# Drop rows without coordinates a small number of historic/dummy entries # Drop rows without coordinates: a small number of historic/dummy entries
# have Easting=0 which would map to the Atlantic. # have Easting=0 which would map to the Atlantic.
df = df.filter( df = df.filter(
pl.col("Easting").is_not_null() pl.col("Easting").is_not_null()

View file

@ -21,7 +21,7 @@ from tqdm import tqdm
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MIN_AREA_SQM = 5_000 # ~70m x 70m skip pocket parks and small ponds MIN_AREA_SQM = 5_000 # ~70m x 70m: skip pocket parks and small ponds
# OSM tags that indicate non-residential green/water areas # OSM tags that indicate non-residential green/water areas
GREENSPACE_TAGS = { GREENSPACE_TAGS = {
@ -108,7 +108,7 @@ class GreenspaceHandler(osmium.SimpleHandler):
return return
# Invalid geometries are often the largest, most complex park/water # Invalid geometries are often the largest, most complex park/water
# multipolygons (self-touching rings from OSM) repair like pois.py # multipolygons (self-touching rings from OSM): repair like pois.py
# rather than silently dropping them. make_valid may return a # rather than silently dropping them. make_valid may return a
# GeometryCollection with stray lines/points; keep only the polygons. # GeometryCollection with stray lines/points; keep only the polygons.
if not geom.is_valid: if not geom.is_valid:
@ -161,7 +161,7 @@ def main():
df.write_parquet(args.output) df.write_parquet(args.output)
print(f"Saved to {args.output}") print(f"Saved to {args.output}")
else: else:
print("No geometries found skipping output") print("No geometries found, skipping output")
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -1,6 +1,6 @@
"""Download Census 2021 children by five-year age band per LSOA. """Download Census 2021 children by five-year age band per LSOA.
Source: NOMIS (ONS Census 2021 TS007A dataset, age by five-year bands) Source: NOMIS (ONS Census 2021, TS007A dataset, age by five-year bands)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
Used to estimate how many primary-age (4-10) and secondary-age (11-15) Used to estimate how many primary-age (4-10) and secondary-age (11-15)

View file

@ -1,6 +1,6 @@
"""Download Census 2021 usual resident population by LSOA. """Download Census 2021 usual resident population by LSOA.
Source: NOMIS (ONS Census 2021 TS001 dataset) Source: NOMIS (ONS Census 2021, TS001 dataset)
License: Open Government Licence v3.0 License: Open Government Licence v3.0
""" """

View file

@ -76,7 +76,7 @@ def test_tenure_lives_rent_free_rolls_into_private_rent():
def test_tenure_percentages_independent_per_lsoa(): def test_tenure_percentages_independent_per_lsoa():
"""Two LSOAs get independent profiles the LSOA granularity is the point.""" """Two LSOAs get independent profiles: the LSOA granularity is the point."""
df = pl.concat( df = pl.concat(
[ [
pl.DataFrame(_long_rows("E01000010", {"Owned: Owns outright": 100})), pl.DataFrame(_long_rows("E01000010", {"Owned: Owns outright": 100})),

View file

@ -76,7 +76,7 @@ def test_clean_national_rail_gtfs_orders_by_stop_sequence_not_file_order(
"""dtd2mysql exports happen to be ordered by stop_sequence within each """dtd2mysql exports happen to be ordered by stop_sequence within each
trip, but nothing guarantees it. Rows arriving out of order must be sorted trip, but nothing guarantees it. Rows arriving out of order must be sorted
by their original stop_sequence before the backwards-time check and the by their original stop_sequence before the backwards-time check and the
0-based renumbering file order would flag the trip as backwards and drop 0-based renumbering. File order would flag the trip as backwards and drop
it (or scramble the stop order).""" it (or scramble the stop order)."""
src = tmp_path / "in.zip" src = tmp_path / "in.zip"
dst = tmp_path / "out.zip" dst = tmp_path / "out.zip"

View file

@ -5,7 +5,7 @@ The right pane shows each crime metric next to its area context: the mean
average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g. average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant ``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
within a postcode (the merge keys it on the postcode), so each postcode within a postcode (the merge keys it on the postcode), so each postcode
contributes its single value weighted by how many properties sit in it keeping contributes its single value weighted by how many properties sit in it, keeping
every scope on the same property-weighted basis as the per-selection mean, so the every scope on the same property-weighted basis as the per-selection mean, so the
four numbers (this selection / sector / outcode / nation) are directly four numbers (this selection / sector / outcode / nation) are directly
comparable. The national figure here is an EXACT property-weighted mean, which is comparable. The national figure here is an EXACT property-weighted mean, which is
@ -18,7 +18,7 @@ crime values from ``postcode.parquet`` and the per-postcode property weights fro
``properties.parquet`` mirrors exactly the two inputs the server loads, so the ``properties.parquet`` mirrors exactly the two inputs the server loads, so the
result matches what the server used to compute (minus its u16 quantization loss). result matches what the server used to compute (minus its u16 quantization loss).
Output schema one row per area: Output schema, one row per area:
scope : ``"national"`` | ``"outcode"`` | ``"sector"`` scope : ``"national"`` | ``"outcode"`` | ``"sector"``
area : the outcode (``"E14"``) / sector (``"E14 2"``); area : the outcode (``"E14"``) / sector (``"E14 2"``);
@ -43,7 +43,7 @@ SCOPE_NATIONAL = "national"
SCOPE_OUTCODE = "outcode" SCOPE_OUTCODE = "outcode"
SCOPE_SECTOR = "sector" SCOPE_SECTOR = "sector"
# Area label on the national row — it spans the whole country, so it has no code. # Area label on the national row. It spans the whole country, so it has no code.
NATIONAL_AREA = "" NATIONAL_AREA = ""
# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g. # Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
@ -71,7 +71,7 @@ def _weighted_mean(column: str) -> pl.Expr:
A null crime value is a genuine gap (the postcode's police force published no A null crime value is a genuine gap (the postcode's police force published no
usable data), not zero crime, so it must dilute neither the numerator nor the usable data), not zero crime, so it must dilute neither the numerator nor the
denominator exactly as the server's former estimator skipped NaN values. denominator, exactly as the server's former estimator skipped NaN values.
Yields null when no postcode in the group has data for this type. Yields null when no postcode in the group has data for this type.
""" """
weight = pl.col(_WEIGHT_COLUMN) weight = pl.col(_WEIGHT_COLUMN)

View file

@ -126,7 +126,7 @@ def transform_crime(
# Sum per-incident weights directly: a 2021 LSOA can receive incidents # Sum per-incident weights directly: a 2021 LSOA can receive incidents
# carrying different `_weight`s in the same month (split 2011 parent at # carrying different `_weight`s in the same month (split 2011 parent at
# 1/N alongside an unsplit one at 1), so `_weight.first() * len` would # 1/N alongside an unsplit one at 1), so `_weight.first() * len` would
# apply one row's weight to all of them and nondeterministically so, # apply one row's weight to all of them, and nondeterministically so,
# since `first` after a join has no ordering guarantee. # since `first` after a join has no ordering guarantee.
filtered.group_by("LSOA code", "year", "Crime type") filtered.group_by("LSOA code", "year", "Crime type")
.agg(pl.col("_weight").sum().alias("count")) .agg(pl.col("_weight").sum().alias("count"))
@ -194,7 +194,7 @@ def _write_crime_by_year(
) )
yearly_per_type = ( yearly_per_type = (
# Per-incident weight sum, not `_weight.first() * len` — see the # Per-incident weight sum, not `_weight.first() * len`. See the
# matching comment in transform_crime. # matching comment in transform_crime.
filtered.group_by("LSOA code", "Crime type", "year") filtered.group_by("LSOA code", "Crime type", "year")
.agg(pl.col("_weight").sum().alias("count")) .agg(pl.col("_weight").sum().alias("count"))

View file

@ -92,7 +92,7 @@ STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
# Trailing-window definitions, in (label, years) form. Each window's average is # Trailing-window definitions, in (label, years) form. Each window's average is
# pooled over the force's covered months inside the window; one average-annual- # pooled over the force's covered months inside the window; one average-annual-
# count column (`(/yr, <label>)`) — the filterable crime feature — is emitted per # count column (`(/yr, <label>)`), the filterable crime feature, is emitted per
# window. # window.
WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2)) WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))

View file

@ -22,20 +22,20 @@ pl.Config.set_tbl_cols(-1)
RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7} RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7}
# Value-quality floor for price aggregations. A flat nominal floor is a blunt # Value-quality floor for price aggregations. A flat nominal floor is a blunt
# tool against a deflating threshold £50k was completely normal for a 1990s # tool against a deflating threshold: £50k was completely normal for a 1990s
# house, so a 50k floor wrongly discarded ~a third of legitimate 1990s # house, so a 50k floor wrongly discarded ~a third of legitimate 1990s
# open-market sales (and deleted properties whose only sales were old/cheap), # open-market sales (and deleted properties whose only sales were old/cheap),
# biasing early-year price history upward. 10k recovers the large [10k,50k) # biasing early-year price history upward. 10k recovers the large [10k,50k)
# band of genuine cheaper sales while still excluding the nominal/junk transfers # band of genuine cheaper sales while still excluding the nominal/junk transfers
# (£1 etc.). A small tail of real sub-10k sales is still dropped a deliberate # (£1 etc.). A small tail of real sub-10k sales is still dropped, a deliberate
# conservative tradeoff to keep clearly-implausible transfers out. # conservative tradeoff to keep clearly-implausible transfers out.
MIN_PRICE = 10_000 MIN_PRICE = 10_000
# Time-aware consecutive-sale jump guard. Price-paid contains keyed-in price # Time-aware consecutive-sale jump guard. Price-paid contains keyed-in price
# errors that pass the MIN_PRICE/category filters — e.g. 13 QUICKSETTS HR2 7PP, # errors that pass the MIN_PRICE/category filters and would otherwise become
# a 93 m² terrace, sold £140,000 in 2016 then "£207,500,000" in 2026 (clearly # latest_price. For example, 13 QUICKSETTS HR2 7PP, a 93 m² terrace, sold
# £207,500 with extra digits, lodged as category A) — and would otherwise # £140,000 in 2016 then "£207,500,000" in 2026 (clearly £207,500 with extra
# become latest_price. A quality sale is flagged when it exceeds its # digits, lodged as category A). A quality sale is flagged when it exceeds its
# neighbouring sale by more than JUMP_TOLERANCE * JUMP_GROWTH_PER_YEAR ** years # neighbouring sale by more than JUMP_TOLERANCE * JUMP_GROWTH_PER_YEAR ** years
# between the two sales. Calibration: genuine extreme appreciation (prime # between the two sales. Calibration: genuine extreme appreciation (prime
# London 1995->2026 is roughly x50 over 31 years) stays comfortably under # London 1995->2026 is roughly x50 over 31 years) stays comfortably under
@ -174,8 +174,8 @@ def _clean_number(column: str, dtype: pl.DataType) -> pl.Expr:
def _join_address_parts(*columns: str) -> pl.Expr: def _join_address_parts(*columns: str) -> pl.Expr:
"""Join address components into one display address, single-spaced. """Join address components into one display address, single-spaced.
Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent
saon is "" on ~88% of rows and ``concat_str(..., ignore_nulls=True)`` (saon is "" on ~88% of rows), and ``concat_str(..., ignore_nulls=True)``
skips only nulls, so empty components still contributed their separator skips only nulls, so empty components still contributed their separator
(``' 10 PALACE GREEN'``, doubled spaces when a middle part was empty). (``' 10 PALACE GREEN'``, doubled spaces when a middle part was empty).
Convert ``''``null per component so ignore_nulls works as intended, then Convert ``''``null per component so ignore_nulls works as intended, then
@ -444,6 +444,13 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
epc_base.sort("inspection_date", descending=True, nulls_last=True) epc_base.sort("inspection_date", descending=True, nulls_last=True)
.group_by("_epc_match_address", "_epc_match_postcode") .group_by("_epc_match_address", "_epc_match_postcode")
.first() .first()
# The deduped row carries the most-recent certificate (sorted newest
# first, .first() keeps it), so normalising its raw tenure here yields
# the LATEST certificate's coarse tenure. Kept past the .drop("tenure")
# so downstream (merge) can tell a still-social dwelling from one that
# was social once but whose latest cert is no longer social. Computed
# before .drop("tenure") because it reads that column.
.with_columns(tenure_status(pl.col("tenure")).alias("latest_tenure_status"))
.drop("tenure") .drop("tenure")
) )
@ -520,7 +527,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
# #
# Emission rule (walking certificates oldest-first, ignoring unknown-tenure # Emission rule (walking certificates oldest-first, ignoring unknown-tenure
# ones so they neither appear nor break the chain): # ones so they neither appear nor break the chain):
# - the first known status is emitted only when it is a rental an # - the first known status is emitted only when it is a rental: an
# owner-occupied baseline is the unremarkable default for a property # owner-occupied baseline is the unremarkable default for a property
# that has changed hands and would only clutter the timeline; # that has changed hands and would only clutter the timeline;
# - every later certificate is emitted when its status differs from the # - every later certificate is emitted when its status differs from the
@ -658,8 +665,8 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
) )
.filter(pl.col("pp_address").is_not_null()) .filter(pl.col("pp_address").is_not_null())
# Price-paid carries ~72k duplicate (address, postcode, date, price) # Price-paid carries ~72k duplicate (address, postcode, date, price)
# transaction groups with DISTINCT transaction ids the same completed # transaction groups with DISTINCT transaction ids (the same completed
# sale lodged twice which double-counted sales in historical_prices. # sale lodged twice), which double-counted sales in historical_prices.
# Collapse each to one row. ppd_category stays in the subset so an # Collapse each to one row. ppd_category stays in the subset so an
# A/B-categorised pair of the same sale survives as two rows; only the # A/B-categorised pair of the same sale survives as two rows; only the
# A row feeds the price aggregations (quality_ok), which is intentional. # A row feeds the price aggregations (quality_ok), which is intentional.
@ -714,6 +721,10 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
pl.col("date_of_transfer").dt.year().alias("year"), pl.col("date_of_transfer").dt.year().alias("year"),
pl.col("date_of_transfer").dt.month().cast(pl.UInt8).alias("month"), pl.col("date_of_transfer").dt.month().cast(pl.UInt8).alias("month"),
"price", "price",
# Per-sale new-build flag straight from the PPD "old_new" field
# (Y = newly built on first transfer, N = established). Kept on
# each transaction so the timeline can mark the new-build sale.
(pl.col("old_new") == "Y").alias("is_new"),
) )
.filter(quality_ok) .filter(quality_ok)
.alias("historical_prices"), .alias("historical_prices"),

View file

@ -83,7 +83,7 @@ _AREA_COLUMNS = [
"% Mixed", "% Mixed",
"% White", "% White",
"% Other", "% Other",
# Crime average annual recorded incident count (incidents/yr), 7-year and # Crime: average annual recorded incident count (incidents/yr), 7-year and
# 2-year windows. These are the filterable crime features; the per-incident # 2-year windows. These are the filterable crime features; the per-incident
# records live in a separate side table the server loads directly (it bypasses # records live in a separate side table the server loads directly (it bypasses
# the merge). # the merge).
@ -145,11 +145,18 @@ _AREA_COLUMNS = [
"% A-levels", "% A-levels",
"% Degree or higher", "% Degree or higher",
"% Other qualifications", "% Other qualifications",
# Tenure (Census 2021 TS054, % of households by tenure) — unlike ethnicity & # Tenure (Census 2021 TS054, % of households by tenure). Unlike ethnicity &
# education these three percentages are user-filterable, not display-only. # education these three percentages are user-filterable, not display-only.
"% Owner occupied", "% Owner occupied",
"% Social rent", "% Social rent",
"% Private rent", "% Private rent",
# Council housing (EPC-derived, NOT Census): share of dwellings in the
# postcode ever recorded as social housing per EPC, and the ever-social
# subset whose latest EPC certificate is no longer social rented (sold off).
# Aggregated from the per-property "was_council_house" / "latest_tenure_status"
# flags in _epc_council_by_postcode and joined onto the AREA frame only.
"% Council housing",
"% Ex-council",
# Politics # Politics
"Voter turnout (%)", "Voter turnout (%)",
"% Labour", "% Labour",
@ -254,7 +261,7 @@ def _subset_numbers_compatible(left: str, right: str) -> bool:
Subset (not equality) is correct ONLY for listed-building name matching: a Subset (not equality) is correct ONLY for listed-building name matching: a
list entry like "10-12 HIGH STREET" should flag "10 HIGH STREET". Address- list entry like "10-12 HIGH STREET" should flag "10 HIGH STREET". Address-
to-address matching must use the canonical `fuzzy_join._numbers_compatible` to-address matching must use the canonical `fuzzy_join._numbers_compatible`
instead (set equality over ``\\d+[A-Z]?`` tokens) subset semantics there instead (set equality over ``\\d+[A-Z]?`` tokens): subset semantics there
let a single flat absorb its whole building (see fuzzy_join docstring). let a single flat absorb its whole building (see fuzzy_join docstring).
""" """
left_nums = set(_NUMBER_RE.findall(left)) left_nums = set(_NUMBER_RE.findall(left))
@ -574,7 +581,7 @@ def _is_current_planning_record(end_date: object) -> bool:
"""A planning record is current when it has no end-date OR its end-date is """A planning record is current when it has no end-date OR its end-date is
still in the future. The planning.data.gov.uk `end-date` field marks when a still in the future. The planning.data.gov.uk `end-date` field marks when a
designation is RETIRED, so a future date (e.g. 2029-12-31) is a still-current designation is RETIRED, so a future date (e.g. 2029-12-31) is a still-current
area and must NOT be dropped the previous "any non-empty date = ended" area and must NOT be dropped. The previous "any non-empty date = ended"
logic wrongly excluded those (e.g. 22 current Gateshead conservation areas).""" logic wrongly excluded those (e.g. 22 current Gateshead conservation areas)."""
if end_date is None: if end_date is None:
return True return True
@ -864,7 +871,7 @@ def _join_area_side_tables(
) -> pl.LazyFrame: ) -> pl.LazyFrame:
base = base.join(iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left") base = base.join(iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left")
# Ethnicity is Census 2021 TS021 at LSOA (~33,755 areas), joined on the same # Ethnicity is Census 2021 TS021 at LSOA (~33,755 areas), joined on the same
# `lsoa21` key as median age and IoD a ~100x granularity gain over the old # `lsoa21` key as median age and IoD, a ~100x granularity gain over the old
# Local-Authority broadcast, with no change to the 6-bucket output schema. # Local-Authority broadcast, with no change to the 6-bucket output schema.
base = base.join(ethnicity, on="lsoa21", how="left") base = base.join(ethnicity, on="lsoa21", how="left")
# Education (Census 2021 TS067 "highest level of qualification") is sourced at # Education (Census 2021 TS067 "highest level of qualification") is sourced at
@ -1140,9 +1147,9 @@ def _address_score(query: str, candidate: str | None, *, allow_token_set: bool)
if not candidate: if not candidate:
return 0 return 0
# token_set_ratio returns 100 whenever the shorter token set is a subset of # token_set_ratio returns 100 whenever the shorter token set is a subset of
# the longer. For a NUMBER-LESS query that is unsafe a single locality # the longer. For a NUMBER-LESS query that is unsafe: a single locality
# token (e.g. "KINGSWOOD") subsets to 100 against any long address that # token (e.g. "KINGSWOOD") subsets to 100 against any long address that
# merely contains it so number-less queries score with token_sort_ratio # merely contains it, so number-less queries score with token_sort_ratio
# only, matching the canonical fuzzy_join._score_bucket. For a NUMBERED # only, matching the canonical fuzzy_join._score_bucket. For a NUMBERED
# query the unconditional fuzzy_join._numbers_compatible gate has already # query the unconditional fuzzy_join._numbers_compatible gate has already
# guaranteed the candidate carries identical house numbers, so token_set # guaranteed the candidate carries identical house numbers, so token_set
@ -1241,14 +1248,14 @@ def _best_listing_match(
``uprn_index`` (postcode-independent, so it is robust even when the ``uprn_index`` (postcode-independent, so it is robust even when the
listing's postcode is slightly off); (2) failing that, the highest listing's postcode is slightly off); (2) failing that, the highest
fuzzy street-address similarity within the listing's own postcode bucket. fuzzy street-address similarity within the listing's own postcode bucket.
No property-attribute heuristics are used `fuzzy_join._numbers_compatible` No property-attribute heuristics are used. `fuzzy_join._numbers_compatible`
gates every fuzzy match unconditionally (so a number-less listing can never gates every fuzzy match unconditionally (so a number-less listing can never
match a numbered property, and vice versa), as in the canonical match a numbered property, and vice versa), as in the canonical
`fuzzy_join._score_bucket`. A house number additionally lowers the score `fuzzy_join._score_bucket`. A house number additionally lowers the score
threshold and (via `_address_score`) permits token_set scoring; a number-less threshold and (via `_address_score`) permits token_set scoring; a number-less
address scores on token_sort only and must match the street almost exactly. address scores on token_sort only and must match the street almost exactly.
The direct-EPC path layers a street-level fallback on top of this strict The direct-EPC path layers a street-level fallback on top of this strict
matcher see `_best_street_epc_fallback`. matcher. See `_best_street_epc_fallback`.
``addressed_fields`` names the candidate columns to fuzzy-match against (a ``addressed_fields`` names the candidate columns to fuzzy-match against (a
candidate may carry both a register and an EPC address). Returns candidate may carry both a register and an EPC address). Returns
@ -1301,7 +1308,7 @@ def _best_listing_match(
# the listing's own postcode unit is the nearest segment of the street, and a # the listing's own postcode unit is the nearest segment of the street, and a
# certificate sharing a house-number token with the listing (e.g. listing # certificate sharing a house-number token with the listing (e.g. listing
# "751 753 Cranbrook Road" vs certificate "751 Cranbrook Road", which fails the # "751 753 Cranbrook Road" vs certificate "751 Cranbrook Road", which fails the
# strict set-equality gate) is almost certainly the right property — both # strict set-equality gate) is almost certainly the right property. Both
# should beat a bare attribute-agreement win. # should beat a bare attribute-agreement win.
_STREET_FALLBACK_SAME_POSTCODE_BONUS = 3.0 _STREET_FALLBACK_SAME_POSTCODE_BONUS = 3.0
_STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0 _STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
@ -1309,8 +1316,8 @@ _STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
# is era-homogeneous (a single development). When the same-street certificates # is era-homogeneous (a single development). When the same-street certificates
# span more than this many years between their oldest and newest build, the # span more than this many years between their oldest and newest build, the
# street mixes construction eras (a Victorian terrace with modern infill, say), # street mixes construction eras (a Victorian terrace with modern infill, say),
# so no single year represents the unidentified number-less listing property # so no single year represents the unidentified number-less listing property.
# the fallback then publishes a null construction year rather than an arbitrary, # The fallback then publishes a null construction year rather than an arbitrary,
# often wrong-by-a-century guess. Two adjacent EPC age bands (one development # often wrong-by-a-century guess. Two adjacent EPC age bands (one development
# straddling a band boundary) span at most ~26 representative years, so this # straddling a band boundary) span at most ~26 representative years, so this
# threshold keeps genuinely-uniform streets while rejecting mixed ones. # threshold keeps genuinely-uniform streets while rejecting mixed ones.
@ -1328,7 +1335,7 @@ def _street_match_with_reliable_construction_year(
``_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS`` between their oldest and newest ``_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS`` between their oldest and newest
build, the street mixes construction eras and the matched certificate's year build, the street mixes construction eras and the matched certificate's year
(e.g. an 1890 Victorian house on a street with 2007 infill) would otherwise (e.g. an 1890 Victorian house on a street with 2007 infill) would otherwise
be presented as the listing property's own so the year and its be presented as the listing property's own, so the year and its
approximate-date flag are nulled, leaving an honest "unknown" rather than a approximate-date flag are nulled, leaving an honest "unknown" rather than a
wrong-by-a-century value. Other street-representative EPC facts (energy wrong-by-a-century value. Other street-representative EPC facts (energy
rating, floor area) are inherently per-property approximations the fallback rating, floor area) are inherently per-property approximations the fallback
@ -1357,7 +1364,7 @@ def _best_street_epc_fallback(
"""Street-level direct-EPC fallback for listings the strict matcher missed. """Street-level direct-EPC fallback for listings the strict matcher missed.
~90% of scraped listings publish a street-level address only ("Oldstead ~90% of scraped listings publish a street-level address only ("Oldstead
Road, Bromley" Rightmove never exposes the house number or UPRN), so the Road, Bromley", since Rightmove never exposes the house number or UPRN), so the
strict matcher in `_best_listing_match` can never match them against the strict matcher in `_best_listing_match` can never match them against the
virtually-always-numbered EPC register and their EPC-derived fields virtually-always-numbered EPC register and their EPC-derived fields
(energy rating, interior height, former-council-house flag, construction (energy rating, interior height, former-council-house flag, construction
@ -1371,7 +1378,7 @@ def _best_street_epc_fallback(
same-postcode-unit preference and a house-number-overlap bonus (a same-postcode-unit preference and a house-number-overlap bonus (a
numbered listing that failed the strict set-equality gate, e.g. a numbered listing that failed the strict set-equality gate, e.g. a
"751 753" range vs "751", still lands on the right property). The result "751 753" range vs "751", still lands on the right property). The result
is street-representative rather than property-exact hence the distinct is street-representative rather than property-exact, hence the distinct
"street" method label so downstream consumers can tell the two confidence "street" method label so downstream consumers can tell the two confidence
levels apart. The matched certificate's construction year is kept only when levels apart. The matched certificate's construction year is kept only when
the street is era-homogeneous (see the street is era-homogeneous (see
@ -1439,7 +1446,7 @@ def _best_street_epc_fallback(
# bathrooms (the upstream storage.py defect noted in # bathrooms (the upstream storage.py defect noted in
# `_finalize_listings`). It systematically over-counts, so comparing # `_finalize_listings`). It systematically over-counts, so comparing
# it to the EPC habitable-room count biases selection toward larger, # it to the EPC habitable-room count biases selection toward larger,
# typically older certificates — the opposite of a useful signal — # typically older certificates (the opposite of a useful signal),
# and there is no clean listing-side habitable-room count to use. # and there is no clean listing-side habitable-room count to use.
if ( if (
listing_postcode listing_postcode
@ -1472,9 +1479,9 @@ def _load_listings_for_merge(listings_path: Path, arcgis_path: Path) -> pl.DataF
"""Read the listings parquet and prepare it for the wide-frame merge. """Read the listings parquet and prepare it for the wide-frame merge.
Output is keyed by `_listing_idx` and carries: Output is keyed by `_listing_idx` and carries:
* `postcode` canonical (NSPL `pcds`) form, with terminated postcodes * `postcode`: canonical (NSPL `pcds`) form, with terminated postcodes
remapped to their nearest active successor; remapped to their nearest active successor;
* `pp_address` the listing's raw register address (used as the * `pp_address`: the listing's raw register address (used as the
address half of the fuzzy match); address half of the fuzzy match);
* one `_actual_*` overlay column per `_LISTING_OVERLAY_SOURCES` entry. * one `_actual_*` overlay column per `_LISTING_OVERLAY_SOURCES` entry.
""" """
@ -1684,7 +1691,7 @@ def _listing_match_frame(listings: pl.DataFrame) -> pl.DataFrame:
Listings are matched to EPC certificates and properties by UPRN and by Listings are matched to EPC certificates and properties by UPRN and by
fuzzy street address within their (now accurate, detail-page-sourced) fuzzy street address within their (now accurate, detail-page-sourced)
postcode never by coordinate proximity so no projected easting/northing postcode (never by coordinate proximity), so no projected easting/northing
is computed here. `_listing_uprn` flows through from the loaded listings. is computed here. `_listing_uprn` flows through from the loaded listings.
""" """
return listings.with_columns( return listings.with_columns(
@ -1763,8 +1770,8 @@ def _index_candidates(
The EPC register's UPRN is NOT unique: a single building/parent UPRN fans The EPC register's UPRN is NOT unique: a single building/parent UPRN fans
across many distinct flats (up to 58 distinct (address, postcode) rows in across many distinct flats (up to 58 distinct (address, postcode) rows in
the 2026-06 data; ~9k UPRNs collide, touching ~20k epc_pp rows). Such a the 2026-06 data; ~9k UPRNs collide, touching ~20k epc_pp rows). Such a
UPRN cannot serve as a 1:1 exact-match key it would mis-link a listing to UPRN cannot serve as a 1:1 exact-match key (it would mis-link a listing to
one arbitrary flat so any UPRN that resolves to more than one distinct one arbitrary flat), so any UPRN that resolves to more than one distinct
``(postcode_key, address_key)`` identity is dropped from ``uprn_index``; ``(postcode_key, address_key)`` identity is dropped from ``uprn_index``;
those listings fall back to the fuzzy street-address matcher, which those listings fall back to the fuzzy street-address matcher, which
disambiguates the specific flat. A UPRN repeated for the SAME identity disambiguates the specific flat. A UPRN repeated for the SAME identity
@ -1878,7 +1885,7 @@ def _index_epc_streets(
maps outcode -> the tokens appearing in at least a quarter of that maps outcode -> the tokens appearing in at least a quarter of that
outcode's street keys. Those are locality suffixes (LONDON, SURREY, the outcode's street keys. Those are locality suffixes (LONDON, SURREY, the
town name) rather than street names, and a fallback match must be anchored town name) rather than street names, and a fallback match must be anchored
by at least one token that is NOT one of them otherwise a town-only by at least one token that is NOT one of them. Otherwise a town-only
listing address ("COULSDON SURREY") token_set-inflates to 100 against any listing address ("COULSDON SURREY") token_set-inflates to 100 against any
street key carrying the same locality suffix and matches an arbitrary street key carrying the same locality suffix and matches an arbitrary
street in the outcode. street in the outcode.
@ -2260,7 +2267,7 @@ def _finalize_listings(df: pl.DataFrame) -> pl.DataFrame:
@dataclass @dataclass
class _BuildResult: class _BuildResult:
"""Outputs of `_build` exactly one of the two slot pairs is populated.""" """Outputs of `_build`: exactly one of the two slot pairs is populated."""
postcode: pl.DataFrame | None = None postcode: pl.DataFrame | None = None
properties: pl.DataFrame | None = None properties: pl.DataFrame | None = None
@ -2286,6 +2293,38 @@ def _fill_property_level_no_defaults(frame: pl.LazyFrame) -> pl.LazyFrame:
) )
def _epc_council_by_postcode(wide: pl.LazyFrame) -> pl.LazyFrame:
"""Aggregate the per-property EPC social-tenure flags to POSTCODE percentages.
Two EPC-derived AREA columns, each a share of *all* deduped dwellings in the
postcode (one row per dwelling in ``wide``). This denominator is the dwelling
universe, NOT Census households: a dwelling with no EPC is ``was_council_house
== "No"`` and so counts against the share, making these EPC-coverage-limited
lower bounds that are not directly comparable to the Census ``% Social rent``
(which stays at LSOA grain). A postcode holds relatively few dwellings, so
these shares are coarse and noisier than an LSOA average.
* ``% Council housing``: dwellings ever recorded as council/social housing
per EPC (``was_council_house == "Yes"``).
* ``% Ex-council``: ever-council dwellings whose LATEST EPC certificate is no
longer social rented (sold off / no longer social):
``was_council_house == "Yes" AND latest_tenure_status != "Rented (social)"``.
A null ``latest_tenure_status`` counts as not-currently-social.
``was_council_house`` is already "Yes"/"No" filled for every row (see
``_fill_property_level_no_defaults``), so the means are over the full postcode.
Returns a postcode-keyed LazyFrame to left-join onto the AREA frame only.
"""
currently_social = (pl.col("latest_tenure_status") == "Rented (social)").fill_null(
False
)
ever_social = pl.col("was_council_house") == "Yes"
return wide.group_by("postcode").agg(
(ever_social.mean() * 100).round(1).alias("% Council housing"),
((ever_social & ~currently_social).mean() * 100).round(1).alias("% Ex-council"),
)
def _build( def _build(
epc_pp_path: Path, epc_pp_path: Path,
arcgis_path: Path, arcgis_path: Path,
@ -2311,9 +2350,9 @@ def _build(
"""Build postcode/properties dataframes (or enriched listings) from epc_pp + auxiliary data. """Build postcode/properties dataframes (or enriched listings) from epc_pp + auxiliary data.
Modes: Modes:
* `normal` produces (postcode_df, properties_df) as before. Ignores * `normal`: produces (postcode_df, properties_df) as before. Ignores
`actual_listings_path` if supplied. `actual_listings_path` if supplied.
* `listings` requires `actual_listings_path`; produces a single * `listings`: requires `actual_listings_path`; produces a single
enriched-listings DataFrame and skips the postcode/properties outputs. enriched-listings DataFrame and skips the postcode/properties outputs.
Listings flow through the same enrichment joins as historical rows, Listings flow through the same enrichment joins as historical rows,
so postcode-scoped features (tree density, crime, deprivation, ) end so postcode-scoped features (tree density, crime, deprivation, ) end
@ -2331,8 +2370,8 @@ def _build(
) )
_validate_lad_source_coverage(iod_path, rental_prices_path) _validate_lad_source_coverage(iod_path, rental_prices_path)
# The dwelling universe floor filter, terminated-postcode remap, # The dwelling universe (floor filter, terminated-postcode remap,
# collapse-dedupe, restrict to active English postcodes is shared with # collapse-dedupe, restrict to active English postcodes) is shared with
# price estimation so estimates line up 1:1 with these rows. See # price estimation so estimates line up 1:1 with these rows. See
# pipeline.transform.property_base. # pipeline.transform.property_base.
wide = build_property_base(epc_pp_path, arcgis_path) wide = build_property_base(epc_pp_path, arcgis_path)
@ -2460,6 +2499,17 @@ def _build(
wide = _join_area_side_tables(wide, **area_side_tables) wide = _join_area_side_tables(wide, **area_side_tables)
postcode_area = _join_area_side_tables(postcode_area, **area_side_tables) postcode_area = _join_area_side_tables(postcode_area, **area_side_tables)
# EPC-derived council/ex-council shares: aggregate the per-property social
# tenure flags to POSTCODE percentages and attach to the AREA frame only
# (these are area columns, like the Census tenure block, not per-property).
# Built before dropping latest_tenure_status, which is its only consumer.
epc_council_by_postcode = _epc_council_by_postcode(wide)
postcode_area = postcode_area.join(epc_council_by_postcode, on="postcode", how="left")
# latest_tenure_status is property-grain and not in _AREA_COLUMNS, so the
# split would otherwise leak it into properties.parquet. It has served its
# purpose (the postcode aggregate above), so drop it from the property frame.
wide = wide.drop("latest_tenure_status", strict=False)
# Derive bedroom count: habitable rooms - 1 (assuming 1 reception room), clipped to 0..4 # Derive bedroom count: habitable rooms - 1 (assuming 1 reception room), clipped to 0..4
wide = wide.with_columns( wide = wide.with_columns(
(pl.col("number_habitable_rooms") - 1) (pl.col("number_habitable_rooms") - 1)
@ -2649,7 +2699,7 @@ def main():
required=False, required=False,
help=( help=(
"Optional scraped-listings parquet. When provided, listings flow " "Optional scraped-listings parquet. When provided, listings flow "
"through the same merge pipeline as historical properties — set " "through the same merge pipeline as historical properties. Set "
"--output-listings to write the enriched-listings file instead " "--output-listings to write the enriched-listings file instead "
"of the postcode/properties files." "of the postcode/properties files."
), ),

View file

@ -43,7 +43,7 @@ GROCERY_STATIC_EXCLUDED_CATEGORIES = {
# Scope: "Public Park Or Garden" is the core park function. "Playing Field" # Scope: "Public Park Or Garden" is the core park function. "Playing Field"
# (open public recreation grounds) is borderline but kept: outside big cities # (open public recreation grounds) is borderline but kept: outside big cities
# the local rec ground is the de facto park. "Play Space" (playgrounds) is # the local rec ground is the de facto park. "Play Space" (playgrounds) is
# excluded a playground is not a park, and "Playground" is already its own # excluded: a playground is not a park, and "Playground" is already its own
# OSM-derived category. The remaining functions (Religious Grounds, Golf # OSM-derived category. The remaining functions (Religious Grounds, Golf
# Course, Cemetery, Allotments, Bowling Green, Tennis Court, Other Sports # Course, Cemetery, Allotments, Bowling Green, Tennis Court, Other Sports
# Facility) are clearly not parks. # Facility) are clearly not parks.
@ -76,7 +76,7 @@ def _groceries_categories(pois: pl.DataFrame) -> list[str]:
with group "Groceries"; it never emits the literal "Supermarket". Collecting with group "Groceries"; it never emits the literal "Supermarket". Collecting
every Groceries category captures both the OSM strings and the brand names. every Groceries category captures both the OSM strings and the brand names.
Speciality food retail (bakeries, butchers, delis, off-licences) is Speciality food retail (bakeries, butchers, delis, off-licences) is
excluded see GROCERY_STATIC_EXCLUDED_CATEGORIES. excluded. See GROCERY_STATIC_EXCLUDED_CATEGORIES.
""" """
if "group" not in pois.columns: if "group" not in pois.columns:
raise ValueError("POI dataframe must include a 'group' column") raise ValueError("POI dataframe must include a 'group' column")
@ -138,7 +138,7 @@ def _greenspace_count_frame(greenspace: pl.DataFrame) -> pl.DataFrame:
os_greenspace.parquet is one row per ACCESS POINT (park gate), which is the os_greenspace.parquet is one row per ACCESS POINT (park gate), which is the
right grain for nearest-distance (the nearest gate is what matters) but right grain for nearest-distance (the nearest gate is what matters) but
wildly over-counts "Number of amenities (Park) within Xkm" a large park wildly over-counts "Number of amenities (Park) within Xkm": a large park
with 30 gates counted as 30 parks. Counting uses one row per site at the with 30 gates counted as 30 parks. Counting uses one row per site at the
site centroid (falling back to the first access point when no centroid is site centroid (falling back to the first access point when no centroid is
available). Degrades gracefully: a legacy parquet without `site_id` is available). Degrades gracefully: a legacy parquet without `site_id` is

View file

@ -125,12 +125,7 @@ async fn static_cache_headers(
response response
} }
/// Add baseline security headers to every response. These are deliberately the
/// low-risk, broadly-compatible set: a Content-Security-Policy and
/// Permissions-Policy are intentionally left out because this app loads many
/// cross-origin resources (maplibre/deck.gl, Stripe, Google Street View,
/// analytics, the error sink) and a mis-scoped policy would break the map or
/// checkout. They should be added later as report-only first.
async fn security_headers( async fn security_headers(
request: axum::extract::Request, request: axum::extract::Request,
next: middleware::Next, next: middleware::Next,

View file

@ -1,6 +1,7 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::{Arc, LazyLock, Mutex}; use std::sync::{Arc, LazyLock, Mutex};
use axum::extract::rejection::JsonRejection;
use axum::extract::{Path, State}; use axum::extract::{Path, State};
use axum::http::StatusCode; use axum::http::StatusCode;
use axum::response::{IntoResponse, Response}; use axum::response::{IntoResponse, Response};
@ -153,7 +154,7 @@ fn current_unix_secs_string() -> String {
/// Fetch the live `is_admin` flag for a user, bypassing any cached token /// Fetch the live `is_admin` flag for a user, bypassing any cached token
/// claims. Returns Err with an HTTP response if PocketBase is unreachable /// claims. Returns Err with an HTTP response if PocketBase is unreachable
/// or returns an unexpected payload — the caller should propagate that. /// or returns an unexpected payload. The caller should propagate that.
async fn verify_is_admin( async fn verify_is_admin(
state: &AppState, state: &AppState,
pb_url: &str, pb_url: &str,
@ -295,7 +296,7 @@ async fn mark_invite_used(
// Defense in depth: PocketBase has no atomic compare-and-swap for record // Defense in depth: PocketBase has no atomic compare-and-swap for record
// updates, and our local + distributed locks could in principle fail (lock // updates, and our local + distributed locks could in principle fail (lock
// server timeout, server restart mid-redemption). Re-read the record and // server timeout, server restart mid-redemption). Re-read the record and
// confirm WE actually own it — if a concurrent redemption beat us to the // confirm WE actually own it. If a concurrent redemption beat us to the
// PATCH, both writes succeeded but the loser's user_id is overwritten and // PATCH, both writes succeeded but the loser's user_id is overwritten and
// we must NOT grant a license. // we must NOT grant a license.
let verify_url = format!("{pb_url}/api/collections/invites/records/{invite_id}"); let verify_url = format!("{pb_url}/api/collections/invites/records/{invite_id}");
@ -328,7 +329,7 @@ async fn mark_invite_used(
invite_id, invite_id,
expected = user_id, expected = user_id,
actual = actual_user, actual = actual_user,
"Invite redemption race lost — invite already claimed by another user" "Invite redemption race lost. Invite already claimed by another user"
); );
return Err((StatusCode::CONFLICT, "Invite was already redeemed").into_response()); return Err((StatusCode::CONFLICT, "Invite was already redeemed").into_response());
} }
@ -472,7 +473,7 @@ pub async fn post_invites(
/// Only recognized when `--dist` is not set (i.e., dev mode). /// Only recognized when `--dist` is not set (i.e., dev mode).
const DEV_INVITE_CODE: &str = "devdevdevdev"; const DEV_INVITE_CODE: &str = "devdevdevdev";
/// Validate an invite code. Public endpoint codes are 12-char random alphanumeric /// Validate an invite code. Public endpoint: codes are 12-char random alphanumeric
/// so enumeration is impractical, and the response only reveals valid/invalid + type. /// so enumeration is impractical, and the response only reveals valid/invalid + type.
pub async fn get_invite( pub async fn get_invite(
State(shared): State<Arc<SharedState>>, State(shared): State<Arc<SharedState>>,
@ -542,7 +543,7 @@ pub async fn get_invite(
let used = !used_by.is_empty(); let used = !used_by.is_empty();
let created_by = invite["created_by"].as_str().unwrap_or(""); let created_by = invite["created_by"].as_str().unwrap_or("");
// Look up inviter's name (email local part) sanitized before returning. // Look up inviter's name (email local part), sanitized before returning.
let invited_by = if !created_by.is_empty() { let invited_by = if !created_by.is_empty() {
let user_url = format!("{pb_url}/api/collections/users/records/{created_by}"); let user_url = format!("{pb_url}/api/collections/users/records/{created_by}");
match state match state
@ -594,7 +595,7 @@ pub async fn get_invite(
pub async fn post_redeem_invite( pub async fn post_redeem_invite(
State(shared): State<Arc<SharedState>>, State(shared): State<Arc<SharedState>>,
Extension(user): Extension<OptionalUser>, Extension(user): Extension<OptionalUser>,
Json(req): Json<RedeemRequest>, body: Result<Json<RedeemRequest>, JsonRejection>,
) -> Response { ) -> Response {
let state = shared.load_state(); let state = shared.load_state();
let user = match user.0 { let user = match user.0 {
@ -602,11 +603,20 @@ pub async fn post_redeem_invite(
None => return StatusCode::UNAUTHORIZED.into_response(), None => return StatusCode::UNAUTHORIZED.into_response(),
}; };
// Parse the body only after authenticating, and return a clean message
// instead of leaking the raw serde rejection (e.g. "missing field `code`").
let req = match body {
Ok(Json(req)) => req,
Err(_) => {
return (StatusCode::BAD_REQUEST, "A valid invite code is required.").into_response()
}
};
if let Err(msg) = validate_invite_code(&req.code) { if let Err(msg) = validate_invite_code(&req.code) {
return (StatusCode::BAD_REQUEST, msg).into_response(); return (StatusCode::BAD_REQUEST, msg).into_response();
} }
// Dev-only: fake redeem — just return "licensed" without touching PocketBase // Dev-only: fake redeem, just return "licensed" without touching PocketBase
if state.is_dev && req.code == DEV_INVITE_CODE { if state.is_dev && req.code == DEV_INVITE_CODE {
info!(user_id = %user.id, "Dev invite redeemed (no-op)"); info!(user_id = %user.id, "Dev invite redeemed (no-op)");
return Json(RedeemResponse { return Json(RedeemResponse {