..
This commit is contained in:
parent
1ee796b282
commit
ab688243d7
36 changed files with 307 additions and 135 deletions
|
|
@ -20,6 +20,7 @@ import { BookmarkIcon } from '../ui/icons/BookmarkIcon';
|
|||
import { TrashIcon } from '../ui/icons/TrashIcon';
|
||||
import { CloseIcon } from '../ui/icons/CloseIcon';
|
||||
import { useLicense } from '../../hooks/useLicense';
|
||||
import { useModalA11y } from '../../hooks/useModalA11y';
|
||||
|
||||
function PageLayout({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
|
|
@ -43,17 +44,43 @@ function DeleteDialog({
|
|||
onConfirm: () => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const dialogRef = useModalA11y();
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onCancel();
|
||||
};
|
||||
document.addEventListener('keydown', handleKeyDown);
|
||||
return () => document.removeEventListener('keydown', handleKeyDown);
|
||||
}, [onCancel]);
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center" onClick={onCancel}>
|
||||
<div className="absolute inset-0 bg-black/50 dark:bg-black/70" />
|
||||
<div
|
||||
className="relative w-full max-w-sm mx-4 bg-white dark:bg-warm-800 rounded-lg shadow-xl border border-warm-200 dark:border-warm-700"
|
||||
className="fixed inset-0 z-50 flex items-center justify-center"
|
||||
onClick={onCancel}
|
||||
role="presentation"
|
||||
>
|
||||
<div className="absolute inset-0 bg-black/50 dark:bg-black/70" aria-hidden="true" />
|
||||
<div
|
||||
ref={dialogRef}
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="delete-search-dialog-title"
|
||||
tabIndex={-1}
|
||||
className="relative w-full max-w-sm mx-4 bg-white dark:bg-warm-800 rounded-lg shadow-xl border border-warm-200 dark:border-warm-700 outline-none"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<div className="flex items-center justify-between px-5 pt-5 pb-3">
|
||||
<h2 className="text-lg font-semibold text-navy-950 dark:text-white">{title}</h2>
|
||||
<h2
|
||||
id="delete-search-dialog-title"
|
||||
className="text-lg font-semibold text-navy-950 dark:text-white"
|
||||
>
|
||||
{title}
|
||||
</h2>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCancel}
|
||||
aria-label={t('common.close')}
|
||||
className="text-warm-400 hover:text-warm-700 dark:text-warm-400 dark:hover:text-warm-200"
|
||||
>
|
||||
<CloseIcon className="w-5 h-5" />
|
||||
|
|
@ -62,12 +89,14 @@ function DeleteDialog({
|
|||
<p className="px-5 pb-4 text-sm text-warm-700 dark:text-warm-300">{message}</p>
|
||||
<div className="flex gap-3 justify-end px-5 pb-5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCancel}
|
||||
className="px-4 py-2 text-sm rounded border border-warm-200 dark:border-warm-700 text-warm-700 dark:text-warm-300 hover:bg-warm-50 dark:hover:bg-warm-700"
|
||||
>
|
||||
{t('common.cancel')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onConfirm}
|
||||
className="px-4 py-2 text-sm rounded bg-red-600 text-white font-medium hover:bg-red-700"
|
||||
>
|
||||
|
|
@ -124,6 +153,7 @@ function NotesInput({ value, onSave }: { value: string; onSave: (notes: string)
|
|||
value={text}
|
||||
onChange={handleChange}
|
||||
onBlur={handleBlur}
|
||||
aria-label={t('savedPage.notesPlaceholder')}
|
||||
placeholder={t('savedPage.notesPlaceholder')}
|
||||
rows={1}
|
||||
className="w-full resize-none overflow-hidden rounded border border-warm-200 dark:border-warm-700 bg-warm-50 dark:bg-warm-900 px-3 py-1.5 text-sm text-warm-700 dark:text-warm-300 placeholder-warm-400 dark:placeholder-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400"
|
||||
|
|
@ -160,6 +190,7 @@ function EditableName({ value, onSave }: { value: string; onSave: (name: string)
|
|||
<input
|
||||
ref={inputRef}
|
||||
value={text}
|
||||
aria-label={t('savedPage.clickToRename')}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') commit();
|
||||
|
|
@ -175,12 +206,15 @@ function EditableName({ value, onSave }: { value: string; onSave: (name: string)
|
|||
}
|
||||
|
||||
return (
|
||||
<h3
|
||||
<h3 className="font-medium text-navy-950 dark:text-warm-100 truncate">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setEditing(true)}
|
||||
className="font-medium text-navy-950 dark:text-warm-100 truncate cursor-pointer hover:text-teal-600 dark:hover:text-teal-400 border-b border-dotted border-transparent hover:border-warm-400 dark:hover:border-warm-500"
|
||||
title={t('savedPage.clickToRename')}
|
||||
className="w-full truncate text-left cursor-pointer hover:text-teal-600 dark:hover:text-teal-400 border-b border-dotted border-transparent hover:border-warm-400 dark:hover:border-warm-500 focus:outline-none focus:ring-1 focus:ring-teal-400 rounded"
|
||||
>
|
||||
{value}
|
||||
</button>
|
||||
</h3>
|
||||
);
|
||||
}
|
||||
|
|
@ -704,7 +738,7 @@ function InviteSection({ user }: { user: AuthUser }) {
|
|||
return next;
|
||||
});
|
||||
} catch {
|
||||
// Silent — non-critical
|
||||
// Silent, non-critical
|
||||
} finally {
|
||||
setInviteHistoryLoading(false);
|
||||
}
|
||||
|
|
@ -853,6 +887,14 @@ export default function AccountPage({
|
|||
const { t } = useTranslation();
|
||||
const [newsletterSaving, setNewsletterSaving] = useState(false);
|
||||
const [newsletterError, setNewsletterError] = useState<string | null>(null);
|
||||
// Mirror the loaded account record's value locally so the checkbox always
|
||||
// reflects the stored value and updates instantly on toggle, instead of the
|
||||
// possibly-stale global auth-store value the async refresh hasn't reconciled
|
||||
// yet (the source of the "checked while stored value is false" flicker).
|
||||
const [newsletterChecked, setNewsletterChecked] = useState(user.newsletter);
|
||||
useEffect(() => {
|
||||
setNewsletterChecked(user.newsletter);
|
||||
}, [user.newsletter]);
|
||||
const { startCheckout, checkingOut, error: checkoutError } = useLicense();
|
||||
const isLicensed = user.subscription === 'licensed' || user.isAdmin;
|
||||
|
||||
|
|
@ -916,10 +958,11 @@ export default function AccountPage({
|
|||
<label className="flex items-center gap-3 cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={user.newsletter}
|
||||
checked={newsletterChecked}
|
||||
disabled={newsletterSaving}
|
||||
onChange={async (e) => {
|
||||
const checked = e.target.checked;
|
||||
setNewsletterChecked(checked); // optimistic; reconciled by onRefreshAuth
|
||||
setNewsletterSaving(true);
|
||||
setNewsletterError(null);
|
||||
try {
|
||||
|
|
@ -933,6 +976,7 @@ export default function AccountPage({
|
|||
assertOk(res, 'Update newsletter');
|
||||
await onRefreshAuth();
|
||||
} catch (err) {
|
||||
setNewsletterChecked(!checked); // revert the optimistic toggle on failure
|
||||
const msg =
|
||||
err instanceof Error ? err.message : t('accountPage.updateNewsletterError');
|
||||
setNewsletterError(msg);
|
||||
|
|
|
|||
63
frontend/src/components/invite/InvitePage.test.tsx
Normal file
63
frontend/src/components/invite/InvitePage.test.tsx
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
// @vitest-environment jsdom
|
||||
import { cleanup, render, screen } from '@testing-library/react';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import InvitePage from './InvitePage';
|
||||
|
||||
vi.mock('react-i18next', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('react-i18next')>()),
|
||||
useTranslation: () => ({ t: (key: string) => key, i18n: { language: 'en' } }),
|
||||
}));
|
||||
|
||||
function mockFetch(invite: { ok: boolean; status: number; body?: unknown }) {
|
||||
return vi.fn((input: RequestInfo | URL) => {
|
||||
const url = String(input);
|
||||
if (url.includes('/api/invite/')) {
|
||||
return Promise.resolve({
|
||||
ok: invite.ok,
|
||||
status: invite.status,
|
||||
json: () => Promise.resolve(invite.body ?? {}),
|
||||
} as Response);
|
||||
}
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: () => Promise.resolve({ current_price_pence: 4999 }),
|
||||
} as Response);
|
||||
});
|
||||
}
|
||||
|
||||
const baseProps = {
|
||||
user: null,
|
||||
theme: 'light' as const,
|
||||
onLoginClick: vi.fn(),
|
||||
onRegisterClick: vi.fn(),
|
||||
onLicenseGranted: vi.fn(),
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('InvitePage invalid-invite messaging', () => {
|
||||
it('shows the unified copy for an unknown code (200 valid:false)', async () => {
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
mockFetch({
|
||||
ok: true,
|
||||
status: 200,
|
||||
body: { valid: false, invite_type: '', used: false, invited_by: null },
|
||||
})
|
||||
);
|
||||
render(<InvitePage code="BOGUS" {...baseProps} />);
|
||||
expect(await screen.findByText('invitePage.invalidInviteLink')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('shows the SAME copy for a malformed code rejected with 400', async () => {
|
||||
vi.stubGlobal('fetch', mockFetch({ ok: false, status: 400 }));
|
||||
render(<InvitePage code="%20" {...baseProps} />);
|
||||
expect(await screen.findByText('invitePage.invalidInviteLink')).toBeTruthy();
|
||||
expect(screen.queryByText('invitePage.invalidInvite')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -115,7 +115,20 @@ export default function InvitePage({
|
|||
fetch(apiUrl(`invite/${encodeURIComponent(code)}`)),
|
||||
fetch(apiUrl('pricing')),
|
||||
]);
|
||||
if (!inviteRes.ok) throw new Error('Failed to validate invite');
|
||||
if (!inviteRes.ok) {
|
||||
// A 4xx means the code itself is malformed or unknown, e.g. /invite/%20
|
||||
// is rejected with 400 while /invite/BOGUS returns 200 {valid:false}.
|
||||
// Render the SAME 'invalid invite link' copy for both so messaging is
|
||||
// consistent; only 5xx / network errors fall through to the transient
|
||||
// error state below.
|
||||
if (inviteRes.status >= 400 && inviteRes.status < 500) {
|
||||
if (!cancelled) {
|
||||
setInvite({ valid: false, invite_type: '', used: false, invited_by: null });
|
||||
}
|
||||
return;
|
||||
}
|
||||
throw new Error('Failed to validate invite');
|
||||
}
|
||||
const data: InviteInfo = await inviteRes.json();
|
||||
if (!cancelled) setInvite(data);
|
||||
if (pricingRes.ok) {
|
||||
|
|
@ -230,7 +243,9 @@ export default function InvitePage({
|
|||
<div className="flex-1 flex items-center justify-center bg-gradient-to-b from-navy-950 via-navy-900 to-navy-900 relative overflow-hidden">
|
||||
<HexCanvas isDark={isDark} />
|
||||
<div className="text-center relative z-10">
|
||||
<p className="text-lg font-medium text-white mb-2">{t('invitePage.invalidInvite')}</p>
|
||||
<p className="text-lg font-medium text-white mb-2">
|
||||
{t('invitePage.couldNotValidateTitle')}
|
||||
</p>
|
||||
<p className="text-warm-400">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import {
|
|||
} from '../../lib/seoLandingPages';
|
||||
import { safeJsonLd } from '../../lib/json-ld';
|
||||
|
||||
const PUBLIC_URL = 'https://perfect-postcode.co.uk';
|
||||
const ProductShowcase = lazy(() => import('../home/ProductShowcase'));
|
||||
|
||||
function ProductShowcaseFallback() {
|
||||
|
|
@ -102,32 +101,11 @@ export default function SeoContentPage({
|
|||
}) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const page = getLocalizedSeoContentPage(pageKey, i18n.language);
|
||||
const url = `${PUBLIC_URL}${page.path}`;
|
||||
usePageMeta(page.metaTitle, page.metaDescription);
|
||||
|
||||
return (
|
||||
<main className="flex-1 overflow-y-auto bg-warm-50 text-navy-950 dark:bg-navy-950 dark:text-warm-100">
|
||||
<FaqJsonLd faq={page.faq} />
|
||||
<JsonLd
|
||||
data={{
|
||||
'@context': 'https://schema.org',
|
||||
'@type': 'BreadcrumbList',
|
||||
itemListElement: [
|
||||
{
|
||||
'@type': 'ListItem',
|
||||
position: 1,
|
||||
name: 'Perfect Postcode',
|
||||
item: `${PUBLIC_URL}/`,
|
||||
},
|
||||
{
|
||||
'@type': 'ListItem',
|
||||
position: 2,
|
||||
name: page.title,
|
||||
item: url,
|
||||
},
|
||||
],
|
||||
}}
|
||||
/>
|
||||
|
||||
<section className="bg-navy-950 text-white">
|
||||
<div className="mx-auto max-w-5xl px-6 py-16 md:px-10 md:py-20">
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { getElectionVoteShareFeatureName } from '../../lib/election-filter';
|
|||
import { getEthnicityFeatureName } from '../../lib/ethnicity-filter';
|
||||
import { getQualificationFeatureName } from '../../lib/qualification-filter';
|
||||
import { getTenureFeatureName } from '../../lib/tenure-filter';
|
||||
import { getCouncilFeatureName } from '../../lib/council-filter';
|
||||
import {
|
||||
POI_DISTANCE_FILTER_NAME,
|
||||
getPoiDistanceFeatureName,
|
||||
|
|
@ -60,6 +61,7 @@ export default memo(function HoverCard({
|
|||
const ethnicityFeatureName = getEthnicityFeatureName(name);
|
||||
const qualificationFeatureName = getQualificationFeatureName(name);
|
||||
const tenureFeatureName = getTenureFeatureName(name);
|
||||
const councilFeatureName = getCouncilFeatureName(name);
|
||||
const poiDistanceFeatureName = getPoiDistanceFeatureName(name);
|
||||
const backendName =
|
||||
schoolBackendName ??
|
||||
|
|
@ -69,6 +71,7 @@ export default memo(function HoverCard({
|
|||
ethnicityFeatureName ??
|
||||
qualificationFeatureName ??
|
||||
tenureFeatureName ??
|
||||
councilFeatureName ??
|
||||
poiDistanceFeatureName ??
|
||||
name;
|
||||
const val = data[`avg_${backendName}`] ?? data[`min_${backendName}`];
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ export default function LocationSearch({
|
|||
return;
|
||||
}
|
||||
|
||||
// Postcode — fetch geometry. Unlike place/address results, a postcode
|
||||
// Postcode: fetch geometry. Unlike place/address results, a postcode
|
||||
// result carries no coordinates, so the camera move genuinely depends on
|
||||
// this response and stays gated by isCurrentLookup.
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -9,11 +9,11 @@ import { MapFallback } from './map-page/Fallbacks';
|
|||
*
|
||||
* deck.gl runs in interleaved mode, sharing MapLibre's WebGL context. When that
|
||||
* shared context gets into a bad state (lost context, a buffer/texture finalized
|
||||
* while still referenced — the "bufferSubData: no buffer" / "bindTexture: deleted
|
||||
* while still referenced, the "bufferSubData: no buffer" / "bindTexture: deleted
|
||||
* object" storm), the next interleaved draw can throw. deck.gl's own animation
|
||||
* loop swallows render errors via its `onError` prop, but interleaved draws fire
|
||||
* from MapLibre's synchronous `render` event, which can run inside a React commit
|
||||
* — so the throw bubbles to the nearest React error boundary.
|
||||
* from MapLibre's synchronous `render` event, which can run inside a React commit,
|
||||
* so the throw bubbles to the nearest React error boundary.
|
||||
*
|
||||
* Without this, that throw reaches the single top-level boundary and blanks the
|
||||
* entire app ("Something went wrong / Refresh the page"). This boundary contains
|
||||
|
|
@ -26,7 +26,7 @@ const MAX_AUTO_RECOVERIES = 2;
|
|||
// Remount after a short delay so a transient GL state can settle first.
|
||||
const RECOVERY_DELAY_MS = 400;
|
||||
// Crashes spaced further apart than this are treated as independent, so the
|
||||
// auto-recovery budget resets — only a tight crash loop escalates to manual retry.
|
||||
// auto-recovery budget resets. Only a tight crash loop escalates to manual retry.
|
||||
const STABILITY_WINDOW_MS = 30_000;
|
||||
|
||||
interface MapErrorBoundaryProps {
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ export default function MobileDrawer({
|
|||
>
|
||||
<div className="h-[10%] shrink-0" aria-hidden="true" />
|
||||
|
||||
{/* Panel — bottom 90% */}
|
||||
{/* Panel: bottom 90% */}
|
||||
<div
|
||||
ref={panelRef}
|
||||
className="pointer-events-auto h-[90%] bg-white dark:bg-navy-950 rounded-t-2xl flex flex-col border-t border-x border-warm-300 ring-1 ring-navy-950/10 shadow-2xl shadow-navy-950/45 dark:border-navy-600 dark:ring-white/10 dark:shadow-black/60 overflow-hidden"
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ describe('computeNumberLineLayout', () => {
|
|||
300,
|
||||
fmt
|
||||
)!;
|
||||
const low = layout.items.find((i) => i.kind === 'national')!; // 30 — lowest
|
||||
const high = layout.items.find((i) => i.kind === 'area')!; // 50 — highest
|
||||
const low = layout.items.find((i) => i.kind === 'national')!; // 30: lowest
|
||||
const high = layout.items.find((i) => i.kind === 'area')!; // 50: highest
|
||||
expect(low.tickX).toBeCloseTo(layout.plotLeft, 5);
|
||||
expect(high.tickX).toBeCloseTo(layout.plotRight, 5);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ export function computeNumberLineLayout(
|
|||
* spanning the lowest→highest value, so the selection can be read against its
|
||||
* reference points (national / outcode / sector) at a glance. Labels are spread
|
||||
* to avoid overlap
|
||||
* — common since the nested area/sector/outcode values cluster — and connected
|
||||
* (common since the nested area/sector/outcode values cluster) and connected
|
||||
* back to their tick with a thin leader line.
|
||||
*/
|
||||
export default function NumberLine({ points, format }: NumberLineProps) {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { getElectionVoteShareFeatureName } from '../../../lib/election-filter';
|
|||
import { getEthnicityFeatureName } from '../../../lib/ethnicity-filter';
|
||||
import { getQualificationFeatureName } from '../../../lib/qualification-filter';
|
||||
import { getTenureFeatureName } from '../../../lib/tenure-filter';
|
||||
import { getCouncilFeatureName } from '../../../lib/council-filter';
|
||||
import { getPoiDistanceFeatureName } from '../../../lib/poi-distance-filter';
|
||||
import { getSchoolBackendFeatureName } from '../../../lib/school-filter';
|
||||
|
||||
|
|
@ -31,6 +32,7 @@ export function getMapPageBackendFeatureName(featureName: string): string {
|
|||
getEthnicityFeatureName(featureName) ??
|
||||
getQualificationFeatureName(featureName) ??
|
||||
getTenureFeatureName(featureName) ??
|
||||
getCouncilFeatureName(featureName) ??
|
||||
getPoiDistanceFeatureName(featureName) ??
|
||||
featureName
|
||||
);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ export const Map = lazy(() => import('../Map'));
|
|||
export const Filters = lazy(() => import('../Filters'));
|
||||
export const POIPane = lazy(() => import('../POIPane'));
|
||||
export const OverlayPane = lazy(() => import('../OverlayPane'));
|
||||
export const ListingPane = lazy(() => import('../ListingPane'));
|
||||
export const AreaPane = lazy(() => import('../AreaPane'));
|
||||
export const PropertiesPane = lazy(() =>
|
||||
import('../PropertiesPane').then((module) => ({ default: module.PropertiesPane }))
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ def match_schools(truth: pl.DataFrame, gias: pl.DataFrame) -> pl.DataFrame:
|
|||
.alias("_pc"),
|
||||
).with_row_index("_row_id")
|
||||
|
||||
# 1. Exact postcode match (unique postcodes only — site-sharing schools
|
||||
# 1. Exact postcode match (unique postcodes only: site-sharing schools
|
||||
# would mismatch phases otherwise; those fall through to name matching).
|
||||
pc_unique = gias.filter(pl.col("_pc").is_not_null()).unique(
|
||||
subset="_pc", keep="none"
|
||||
|
|
@ -291,7 +291,7 @@ def main() -> None:
|
|||
print(f"\nWrote matched rows to {args.matched_out}")
|
||||
|
||||
if binding.is_empty():
|
||||
raise SystemExit("No binding, matchable cutoffs — nothing to calibrate on")
|
||||
raise SystemExit("No binding, matchable cutoffs: nothing to calibrate on")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ computation failed or was interrupted, leaving either zero rows or only
|
|||
the origin postcode. We detect this by an absolute, structural criterion:
|
||||
a file is corrupt only when it is unreadable or has a row count at or below
|
||||
CORRUPT_ROW_FLOOR. Per-mode percentile/median/range figures are reported
|
||||
for context only — they never drive the deletable set, so repeated runs
|
||||
for context only. They never drive the deletable set, so repeated runs
|
||||
(including with --delete) are idempotent and never erode legitimate
|
||||
small-catchment (rural/island) origins.
|
||||
|
||||
Duplicates arise when places.parquet is rebuilt between R5 runs — each
|
||||
Duplicates arise when places.parquet is rebuilt between R5 runs. Each
|
||||
place gets a new numeric index prefix, so the skip-completed logic
|
||||
doesn't recognize previous results. --dedup keeps only the largest
|
||||
file per slug and removes the rest.
|
||||
|
|
@ -105,7 +105,7 @@ def find_bad_files(base_dir: Path) -> tuple[list[BadFile], dict[str, dict]]:
|
|||
if not row_counts:
|
||||
continue
|
||||
|
||||
# Reporting statistics only — these never decide what gets deleted.
|
||||
# Reporting statistics only; these never decide what gets deleted.
|
||||
p5 = percentile(row_counts, 5)
|
||||
median = percentile(row_counts, 50)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/resea
|
|||
|
||||
# Pre-staged file path. Ofcom put the entire ofcom.org.uk domain behind
|
||||
# Cloudflare's Managed Challenge in 2026, which requires a JS-executing
|
||||
# browser to pass — no amount of User-Agent / TLS-impersonation spoofing
|
||||
# browser to pass. No amount of User-Agent / TLS-impersonation spoofing
|
||||
# (curl_cffi chrome120..131, safari17, firefox133, chrome_android) gets
|
||||
# past it. When the automated download fails, the user must download the
|
||||
# zip manually from the Source URL above and place it at this path.
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ def _fetch_csv(
|
|||
"""Download a CSV, defending against silent truncation.
|
||||
|
||||
The NOMIS bulk files are served with chunked transfer encoding and no
|
||||
Content-Length, so a dropped connection ends the stream without raising —
|
||||
Content-Length, so a dropped connection ends the stream without raising,
|
||||
yielding a short file. We retry until the parsed row count reaches the
|
||||
file's known approximate size, keeping the largest parse seen.
|
||||
"""
|
||||
|
|
@ -72,7 +72,7 @@ def _fetch_csv(
|
|||
try:
|
||||
download(url, dest)
|
||||
df = pl.read_csv(dest)
|
||||
except Exception as exc: # noqa: BLE001 — retry any transport/parse error
|
||||
except Exception as exc: # noqa: BLE001: retry any transport/parse error
|
||||
print(f" {url} attempt {attempt}: error {exc}")
|
||||
time.sleep(3)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ def select_coverage_archives(
|
|||
adjacent one) matters when the adjacent snapshot is missing from the index:
|
||||
skipping it would leave a multi-month hole, while overlap only costs
|
||||
download time because extraction skips already-extracted months. A hole no
|
||||
archive can bridge is a publication gap in the source — a hard error unless
|
||||
archive can bridge is a publication gap in the source: a hard error unless
|
||||
``allow_gaps``, since the run would otherwise be stamped complete with
|
||||
artificial dips in every crime-over-time series.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -143,8 +143,8 @@ def _has_dwellings(min_d: int | None, max_d: int | None) -> bool:
|
|||
def _brownfield_url(entity) -> str | None:
|
||||
"""Canonical per-site page on the Planning Data platform.
|
||||
|
||||
The register's own ``site-plan-url`` is unreliable — many LPAs point every
|
||||
row at a single generic register landing page — so we link to the stable,
|
||||
The register's own ``site-plan-url`` is unreliable (many LPAs point every
|
||||
row at a single generic register landing page), so we link to the stable,
|
||||
always-per-site entity page instead.
|
||||
"""
|
||||
entity_id = _to_int(entity)
|
||||
|
|
|
|||
|
|
@ -10,13 +10,13 @@ rather than a single headline number.
|
|||
We give the ONS bands colloquial labels (the census's "Level 1/2/3/4+" jargon
|
||||
means little to a homebuyer): No qualifications / Some GCSEs / Good GCSEs /
|
||||
Apprenticeship / A-levels / Degree or higher / Other qualifications. NOTE the
|
||||
census does NOT split undergraduate from postgraduate — "Level 4 and above" is a
|
||||
census does NOT split undergraduate from postgraduate. "Level 4 and above" is a
|
||||
single bucket ("Degree or higher").
|
||||
|
||||
The join key downstream (merge.py) is `lsoa21`, the same key used for ethnicity,
|
||||
median age, and IoD.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS067 dataset, NM_2084_1)
|
||||
Source: NOMIS (ONS Census 2021, TS067 dataset, NM_2084_1)
|
||||
License: Open Government Licence v3.0
|
||||
"""
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ BASE_URL = (
|
|||
)
|
||||
|
||||
# Map each canonical NOMIS C2021_HIQUAL_8 band to our colloquial output bucket.
|
||||
# 1:1 mapping (no folding) — keyed on the exact NOMIS label so a relabelled or
|
||||
# 1:1 mapping (no folding): keyed on the exact NOMIS label so a relabelled or
|
||||
# missing band fails loudly in validation instead of silently dropping people.
|
||||
BAND_MAP = {
|
||||
"No qualifications": "No qualifications",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import httpx
|
|||
import polars as pl
|
||||
|
||||
# UK Parliament publishes candidate-level results for the 2024 General Election.
|
||||
# One row per candidate per constituency — we aggregate to per-constituency stats.
|
||||
# One row per candidate per constituency. We aggregate to per-constituency stats.
|
||||
URL = "https://electionresults.parliament.uk/general-elections/6/candidacies.csv"
|
||||
|
||||
# Map party names to a smaller set for vote share columns.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from pathlib import Path
|
|||
|
||||
import httpx
|
||||
|
||||
# ArcGIS REST API — query for England only, generalised (BGC) resolution
|
||||
# ArcGIS REST API: query for England only, generalised (BGC) resolution
|
||||
URL = (
|
||||
"https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/"
|
||||
"Countries_December_2024_Boundaries_UK_BGC/FeatureServer/0/query"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ different neighbourhoods in one borough no longer share an identical ethnicity
|
|||
profile. The join key downstream (merge.py) is `lsoa21`, the same key already
|
||||
used for median age and IoD.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS021 dataset, NM_2041_1)
|
||||
Source: NOMIS (ONS Census 2021, TS021 dataset, NM_2041_1)
|
||||
License: Open Government Licence v3.0
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ GIAS is the DfE register of all educational establishments in England, updated
|
|||
daily. The CSV is generated on-demand via a four-step interaction with the
|
||||
public Downloads page (there is no static URL):
|
||||
|
||||
1. GET /Downloads — extract anti-forgery token, the `all.edubase.data` tag,
|
||||
1. GET /Downloads: extract anti-forgery token, the `all.edubase.data` tag,
|
||||
and the FileGeneratedDate that the server expects for that tag today.
|
||||
2. POST /Downloads/Collate — submit the form to start file generation. The
|
||||
2. POST /Downloads/Collate: submit the form to start file generation. The
|
||||
redirect URL contains a generation UUID.
|
||||
3. Poll /Downloads/GenerateAjax/{id} until status:true.
|
||||
4. GET the Azure blob URL with ?id={id} — returns a ZIP containing
|
||||
4. GET the Azure blob URL with ?id={id}: returns a ZIP containing
|
||||
`edubasealldataYYYYMMDD.csv`.
|
||||
|
||||
The CSV is cp1252-encoded with 135 columns. We keep the fields useful for a
|
||||
|
|
@ -262,7 +262,7 @@ def transform(zip_bytes: bytes) -> pl.DataFrame:
|
|||
.cast(pl.Float32, strict=False),
|
||||
)
|
||||
|
||||
# Drop rows without coordinates — a small number of historic/dummy entries
|
||||
# Drop rows without coordinates: a small number of historic/dummy entries
|
||||
# have Easting=0 which would map to the Atlantic.
|
||||
df = df.filter(
|
||||
pl.col("Easting").is_not_null()
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from tqdm import tqdm
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
MIN_AREA_SQM = 5_000 # ~70m x 70m — skip pocket parks and small ponds
|
||||
MIN_AREA_SQM = 5_000 # ~70m x 70m: skip pocket parks and small ponds
|
||||
|
||||
# OSM tags that indicate non-residential green/water areas
|
||||
GREENSPACE_TAGS = {
|
||||
|
|
@ -108,7 +108,7 @@ class GreenspaceHandler(osmium.SimpleHandler):
|
|||
return
|
||||
|
||||
# Invalid geometries are often the largest, most complex park/water
|
||||
# multipolygons (self-touching rings from OSM) — repair like pois.py
|
||||
# multipolygons (self-touching rings from OSM): repair like pois.py
|
||||
# rather than silently dropping them. make_valid may return a
|
||||
# GeometryCollection with stray lines/points; keep only the polygons.
|
||||
if not geom.is_valid:
|
||||
|
|
@ -161,7 +161,7 @@ def main():
|
|||
df.write_parquet(args.output)
|
||||
print(f"Saved to {args.output}")
|
||||
else:
|
||||
print("No geometries found — skipping output")
|
||||
print("No geometries found, skipping output")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Download Census 2021 children by five-year age band per LSOA.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS007A dataset, age by five-year bands)
|
||||
Source: NOMIS (ONS Census 2021, TS007A dataset, age by five-year bands)
|
||||
License: Open Government Licence v3.0
|
||||
|
||||
Used to estimate how many primary-age (4-10) and secondary-age (11-15)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Download Census 2021 usual resident population by LSOA.
|
||||
|
||||
Source: NOMIS (ONS Census 2021 — TS001 dataset)
|
||||
Source: NOMIS (ONS Census 2021, TS001 dataset)
|
||||
License: Open Government Licence v3.0
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ def test_tenure_lives_rent_free_rolls_into_private_rent():
|
|||
|
||||
|
||||
def test_tenure_percentages_independent_per_lsoa():
|
||||
"""Two LSOAs get independent profiles — the LSOA granularity is the point."""
|
||||
"""Two LSOAs get independent profiles: the LSOA granularity is the point."""
|
||||
df = pl.concat(
|
||||
[
|
||||
pl.DataFrame(_long_rows("E01000010", {"Owned: Owns outright": 100})),
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ def test_clean_national_rail_gtfs_orders_by_stop_sequence_not_file_order(
|
|||
"""dtd2mysql exports happen to be ordered by stop_sequence within each
|
||||
trip, but nothing guarantees it. Rows arriving out of order must be sorted
|
||||
by their original stop_sequence before the backwards-time check and the
|
||||
0-based renumbering — file order would flag the trip as backwards and drop
|
||||
0-based renumbering. File order would flag the trip as backwards and drop
|
||||
it (or scramble the stop order)."""
|
||||
src = tmp_path / "in.zip"
|
||||
dst = tmp_path / "out.zip"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ The right pane shows each crime metric next to its area context: the mean
|
|||
average-annual count (``"X (/yr, 7y)"``) across the selection's postcode sector (e.g.
|
||||
``"E14 2"``), its outcode (e.g. ``"E14"``), and the nation. Crime is constant
|
||||
within a postcode (the merge keys it on the postcode), so each postcode
|
||||
contributes its single value weighted by how many properties sit in it — keeping
|
||||
contributes its single value weighted by how many properties sit in it, keeping
|
||||
every scope on the same property-weighted basis as the per-selection mean, so the
|
||||
four numbers (this selection / sector / outcode / nation) are directly
|
||||
comparable. The national figure here is an EXACT property-weighted mean, which is
|
||||
|
|
@ -18,7 +18,7 @@ crime values from ``postcode.parquet`` and the per-postcode property weights fro
|
|||
``properties.parquet`` mirrors exactly the two inputs the server loads, so the
|
||||
result matches what the server used to compute (minus its u16 quantization loss).
|
||||
|
||||
Output schema — one row per area:
|
||||
Output schema, one row per area:
|
||||
|
||||
scope : ``"national"`` | ``"outcode"`` | ``"sector"``
|
||||
area : the outcode (``"E14"``) / sector (``"E14 2"``);
|
||||
|
|
@ -43,7 +43,7 @@ SCOPE_NATIONAL = "national"
|
|||
SCOPE_OUTCODE = "outcode"
|
||||
SCOPE_SECTOR = "sector"
|
||||
|
||||
# Area label on the national row — it spans the whole country, so it has no code.
|
||||
# Area label on the national row. It spans the whole country, so it has no code.
|
||||
NATIONAL_AREA = ""
|
||||
|
||||
# Both merge outputs key on the canonical NSPL `pcds` postcode (spaced, e.g.
|
||||
|
|
@ -71,7 +71,7 @@ def _weighted_mean(column: str) -> pl.Expr:
|
|||
|
||||
A null crime value is a genuine gap (the postcode's police force published no
|
||||
usable data), not zero crime, so it must dilute neither the numerator nor the
|
||||
denominator — exactly as the server's former estimator skipped NaN values.
|
||||
denominator, exactly as the server's former estimator skipped NaN values.
|
||||
Yields null when no postcode in the group has data for this type.
|
||||
"""
|
||||
weight = pl.col(_WEIGHT_COLUMN)
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ def transform_crime(
|
|||
# Sum per-incident weights directly: a 2021 LSOA can receive incidents
|
||||
# carrying different `_weight`s in the same month (split 2011 parent at
|
||||
# 1/N alongside an unsplit one at 1), so `_weight.first() * len` would
|
||||
# apply one row's weight to all of them — and nondeterministically so,
|
||||
# apply one row's weight to all of them, and nondeterministically so,
|
||||
# since `first` after a join has no ordering guarantee.
|
||||
filtered.group_by("LSOA code", "year", "Crime type")
|
||||
.agg(pl.col("_weight").sum().alias("count"))
|
||||
|
|
@ -194,7 +194,7 @@ def _write_crime_by_year(
|
|||
)
|
||||
|
||||
yearly_per_type = (
|
||||
# Per-incident weight sum, not `_weight.first() * len` — see the
|
||||
# Per-incident weight sum, not `_weight.first() * len`. See the
|
||||
# matching comment in transform_crime.
|
||||
filtered.group_by("LSOA code", "Crime type", "year")
|
||||
.agg(pl.col("_weight").sum().alias("count"))
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ STREET_CSV_NAME_RE = re.compile(r"^(\d{4}-\d{2})-(.+)-street\.csv$")
|
|||
|
||||
# Trailing-window definitions, in (label, years) form. Each window's average is
|
||||
# pooled over the force's covered months inside the window; one average-annual-
|
||||
# count column (`(/yr, <label>)`) — the filterable crime feature — is emitted per
|
||||
# count column (`(/yr, <label>)`), the filterable crime feature, is emitted per
|
||||
# window.
|
||||
WINDOWS: tuple[tuple[str, int], ...] = (("7y", 7), ("2y", 2))
|
||||
|
||||
|
|
|
|||
|
|
@ -22,20 +22,20 @@ pl.Config.set_tbl_cols(-1)
|
|||
|
||||
RATING_RANK = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7}
|
||||
# Value-quality floor for price aggregations. A flat nominal floor is a blunt
|
||||
# tool against a deflating threshold — £50k was completely normal for a 1990s
|
||||
# tool against a deflating threshold: £50k was completely normal for a 1990s
|
||||
# house, so a 50k floor wrongly discarded ~a third of legitimate 1990s
|
||||
# open-market sales (and deleted properties whose only sales were old/cheap),
|
||||
# biasing early-year price history upward. 10k recovers the large [10k,50k)
|
||||
# band of genuine cheaper sales while still excluding the nominal/junk transfers
|
||||
# (£1 etc.). A small tail of real sub-10k sales is still dropped — a deliberate
|
||||
# (£1 etc.). A small tail of real sub-10k sales is still dropped, a deliberate
|
||||
# conservative tradeoff to keep clearly-implausible transfers out.
|
||||
MIN_PRICE = 10_000
|
||||
|
||||
# Time-aware consecutive-sale jump guard. Price-paid contains keyed-in price
|
||||
# errors that pass the MIN_PRICE/category filters — e.g. 13 QUICKSETTS HR2 7PP,
|
||||
# a 93 m² terrace, sold £140,000 in 2016 then "£207,500,000" in 2026 (clearly
|
||||
# £207,500 with extra digits, lodged as category A) — and would otherwise
|
||||
# become latest_price. A quality sale is flagged when it exceeds its
|
||||
# errors that pass the MIN_PRICE/category filters and would otherwise become
|
||||
# latest_price. For example, 13 QUICKSETTS HR2 7PP, a 93 m² terrace, sold
|
||||
# £140,000 in 2016 then "£207,500,000" in 2026 (clearly £207,500 with extra
|
||||
# digits, lodged as category A). A quality sale is flagged when it exceeds its
|
||||
# neighbouring sale by more than JUMP_TOLERANCE * JUMP_GROWTH_PER_YEAR ** years
|
||||
# between the two sales. Calibration: genuine extreme appreciation (prime
|
||||
# London 1995->2026 is roughly x50 over 31 years) stays comfortably under
|
||||
|
|
@ -174,8 +174,8 @@ def _clean_number(column: str, dtype: pl.DataType) -> pl.Expr:
|
|||
def _join_address_parts(*columns: str) -> pl.Expr:
|
||||
"""Join address components into one display address, single-spaced.
|
||||
|
||||
Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent —
|
||||
saon is "" on ~88% of rows — and ``concat_str(..., ignore_nulls=True)``
|
||||
Price-paid SAON/PAON/STREET are EMPTY STRINGS (not null) when absent
|
||||
(saon is "" on ~88% of rows), and ``concat_str(..., ignore_nulls=True)``
|
||||
skips only nulls, so empty components still contributed their separator
|
||||
(``' 10 PALACE GREEN'``, doubled spaces when a middle part was empty).
|
||||
Convert ``''``→null per component so ignore_nulls works as intended, then
|
||||
|
|
@ -444,6 +444,13 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
|
|||
epc_base.sort("inspection_date", descending=True, nulls_last=True)
|
||||
.group_by("_epc_match_address", "_epc_match_postcode")
|
||||
.first()
|
||||
# The deduped row carries the most-recent certificate (sorted newest
|
||||
# first, .first() keeps it), so normalising its raw tenure here yields
|
||||
# the LATEST certificate's coarse tenure. Kept past the .drop("tenure")
|
||||
# so downstream (merge) can tell a still-social dwelling from one that
|
||||
# was social once but whose latest cert is no longer social. Computed
|
||||
# before .drop("tenure") because it reads that column.
|
||||
.with_columns(tenure_status(pl.col("tenure")).alias("latest_tenure_status"))
|
||||
.drop("tenure")
|
||||
)
|
||||
|
||||
|
|
@ -520,7 +527,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
|
|||
#
|
||||
# Emission rule (walking certificates oldest-first, ignoring unknown-tenure
|
||||
# ones so they neither appear nor break the chain):
|
||||
# - the first known status is emitted only when it is a rental — an
|
||||
# - the first known status is emitted only when it is a rental: an
|
||||
# owner-occupied baseline is the unremarkable default for a property
|
||||
# that has changed hands and would only clutter the timeline;
|
||||
# - every later certificate is emitted when its status differs from the
|
||||
|
|
@ -658,8 +665,8 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
|
|||
)
|
||||
.filter(pl.col("pp_address").is_not_null())
|
||||
# Price-paid carries ~72k duplicate (address, postcode, date, price)
|
||||
# transaction groups with DISTINCT transaction ids — the same completed
|
||||
# sale lodged twice — which double-counted sales in historical_prices.
|
||||
# transaction groups with DISTINCT transaction ids (the same completed
|
||||
# sale lodged twice), which double-counted sales in historical_prices.
|
||||
# Collapse each to one row. ppd_category stays in the subset so an
|
||||
# A/B-categorised pair of the same sale survives as two rows; only the
|
||||
# A row feeds the price aggregations (quality_ok), which is intentional.
|
||||
|
|
@ -714,6 +721,10 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
|
|||
pl.col("date_of_transfer").dt.year().alias("year"),
|
||||
pl.col("date_of_transfer").dt.month().cast(pl.UInt8).alias("month"),
|
||||
"price",
|
||||
# Per-sale new-build flag straight from the PPD "old_new" field
|
||||
# (Y = newly built on first transfer, N = established). Kept on
|
||||
# each transaction so the timeline can mark the new-build sale.
|
||||
(pl.col("old_new") == "Y").alias("is_new"),
|
||||
)
|
||||
.filter(quality_ok)
|
||||
.alias("historical_prices"),
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ _AREA_COLUMNS = [
|
|||
"% Mixed",
|
||||
"% White",
|
||||
"% Other",
|
||||
# Crime — average annual recorded incident count (incidents/yr), 7-year and
|
||||
# Crime: average annual recorded incident count (incidents/yr), 7-year and
|
||||
# 2-year windows. These are the filterable crime features; the per-incident
|
||||
# records live in a separate side table the server loads directly (it bypasses
|
||||
# the merge).
|
||||
|
|
@ -145,11 +145,18 @@ _AREA_COLUMNS = [
|
|||
"% A-levels",
|
||||
"% Degree or higher",
|
||||
"% Other qualifications",
|
||||
# Tenure (Census 2021 TS054, % of households by tenure) — unlike ethnicity &
|
||||
# Tenure (Census 2021 TS054, % of households by tenure). Unlike ethnicity &
|
||||
# education these three percentages are user-filterable, not display-only.
|
||||
"% Owner occupied",
|
||||
"% Social rent",
|
||||
"% Private rent",
|
||||
# Council housing (EPC-derived, NOT Census): share of dwellings in the
|
||||
# postcode ever recorded as social housing per EPC, and the ever-social
|
||||
# subset whose latest EPC certificate is no longer social rented (sold off).
|
||||
# Aggregated from the per-property "was_council_house" / "latest_tenure_status"
|
||||
# flags in _epc_council_by_postcode and joined onto the AREA frame only.
|
||||
"% Council housing",
|
||||
"% Ex-council",
|
||||
# Politics
|
||||
"Voter turnout (%)",
|
||||
"% Labour",
|
||||
|
|
@ -254,7 +261,7 @@ def _subset_numbers_compatible(left: str, right: str) -> bool:
|
|||
Subset (not equality) is correct ONLY for listed-building name matching: a
|
||||
list entry like "10-12 HIGH STREET" should flag "10 HIGH STREET". Address-
|
||||
to-address matching must use the canonical `fuzzy_join._numbers_compatible`
|
||||
instead (set equality over ``\\d+[A-Z]?`` tokens) — subset semantics there
|
||||
instead (set equality over ``\\d+[A-Z]?`` tokens): subset semantics there
|
||||
let a single flat absorb its whole building (see fuzzy_join docstring).
|
||||
"""
|
||||
left_nums = set(_NUMBER_RE.findall(left))
|
||||
|
|
@ -574,7 +581,7 @@ def _is_current_planning_record(end_date: object) -> bool:
|
|||
"""A planning record is current when it has no end-date OR its end-date is
|
||||
still in the future. The planning.data.gov.uk `end-date` field marks when a
|
||||
designation is RETIRED, so a future date (e.g. 2029-12-31) is a still-current
|
||||
area and must NOT be dropped — the previous "any non-empty date = ended"
|
||||
area and must NOT be dropped. The previous "any non-empty date = ended"
|
||||
logic wrongly excluded those (e.g. 22 current Gateshead conservation areas)."""
|
||||
if end_date is None:
|
||||
return True
|
||||
|
|
@ -864,7 +871,7 @@ def _join_area_side_tables(
|
|||
) -> pl.LazyFrame:
|
||||
base = base.join(iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left")
|
||||
# Ethnicity is Census 2021 TS021 at LSOA (~33,755 areas), joined on the same
|
||||
# `lsoa21` key as median age and IoD — a ~100x granularity gain over the old
|
||||
# `lsoa21` key as median age and IoD, a ~100x granularity gain over the old
|
||||
# Local-Authority broadcast, with no change to the 6-bucket output schema.
|
||||
base = base.join(ethnicity, on="lsoa21", how="left")
|
||||
# Education (Census 2021 TS067 "highest level of qualification") is sourced at
|
||||
|
|
@ -1140,9 +1147,9 @@ def _address_score(query: str, candidate: str | None, *, allow_token_set: bool)
|
|||
if not candidate:
|
||||
return 0
|
||||
# token_set_ratio returns 100 whenever the shorter token set is a subset of
|
||||
# the longer. For a NUMBER-LESS query that is unsafe — a single locality
|
||||
# the longer. For a NUMBER-LESS query that is unsafe: a single locality
|
||||
# token (e.g. "KINGSWOOD") subsets to 100 against any long address that
|
||||
# merely contains it — so number-less queries score with token_sort_ratio
|
||||
# merely contains it, so number-less queries score with token_sort_ratio
|
||||
# only, matching the canonical fuzzy_join._score_bucket. For a NUMBERED
|
||||
# query the unconditional fuzzy_join._numbers_compatible gate has already
|
||||
# guaranteed the candidate carries identical house numbers, so token_set
|
||||
|
|
@ -1241,14 +1248,14 @@ def _best_listing_match(
|
|||
``uprn_index`` (postcode-independent, so it is robust even when the
|
||||
listing's postcode is slightly off); (2) failing that, the highest
|
||||
fuzzy street-address similarity within the listing's own postcode bucket.
|
||||
No property-attribute heuristics are used — `fuzzy_join._numbers_compatible`
|
||||
No property-attribute heuristics are used. `fuzzy_join._numbers_compatible`
|
||||
gates every fuzzy match unconditionally (so a number-less listing can never
|
||||
match a numbered property, and vice versa), as in the canonical
|
||||
`fuzzy_join._score_bucket`. A house number additionally lowers the score
|
||||
threshold and (via `_address_score`) permits token_set scoring; a number-less
|
||||
address scores on token_sort only and must match the street almost exactly.
|
||||
The direct-EPC path layers a street-level fallback on top of this strict
|
||||
matcher — see `_best_street_epc_fallback`.
|
||||
matcher. See `_best_street_epc_fallback`.
|
||||
|
||||
``addressed_fields`` names the candidate columns to fuzzy-match against (a
|
||||
candidate may carry both a register and an EPC address). Returns
|
||||
|
|
@ -1301,7 +1308,7 @@ def _best_listing_match(
|
|||
# the listing's own postcode unit is the nearest segment of the street, and a
|
||||
# certificate sharing a house-number token with the listing (e.g. listing
|
||||
# "751 753 Cranbrook Road" vs certificate "751 Cranbrook Road", which fails the
|
||||
# strict set-equality gate) is almost certainly the right property — both
|
||||
# strict set-equality gate) is almost certainly the right property. Both
|
||||
# should beat a bare attribute-agreement win.
|
||||
_STREET_FALLBACK_SAME_POSTCODE_BONUS = 3.0
|
||||
_STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
|
||||
|
|
@ -1309,8 +1316,8 @@ _STREET_FALLBACK_NUMBER_OVERLAP_BONUS = 8.0
|
|||
# is era-homogeneous (a single development). When the same-street certificates
|
||||
# span more than this many years between their oldest and newest build, the
|
||||
# street mixes construction eras (a Victorian terrace with modern infill, say),
|
||||
# so no single year represents the unidentified number-less listing property —
|
||||
# the fallback then publishes a null construction year rather than an arbitrary,
|
||||
# so no single year represents the unidentified number-less listing property.
|
||||
# The fallback then publishes a null construction year rather than an arbitrary,
|
||||
# often wrong-by-a-century guess. Two adjacent EPC age bands (one development
|
||||
# straddling a band boundary) span at most ~26 representative years, so this
|
||||
# threshold keeps genuinely-uniform streets while rejecting mixed ones.
|
||||
|
|
@ -1328,7 +1335,7 @@ def _street_match_with_reliable_construction_year(
|
|||
``_STREET_FALLBACK_CONSTRUCTION_SPAN_YEARS`` between their oldest and newest
|
||||
build, the street mixes construction eras and the matched certificate's year
|
||||
(e.g. an 1890 Victorian house on a street with 2007 infill) would otherwise
|
||||
be presented as the listing property's own — so the year and its
|
||||
be presented as the listing property's own, so the year and its
|
||||
approximate-date flag are nulled, leaving an honest "unknown" rather than a
|
||||
wrong-by-a-century value. Other street-representative EPC facts (energy
|
||||
rating, floor area) are inherently per-property approximations the fallback
|
||||
|
|
@ -1357,7 +1364,7 @@ def _best_street_epc_fallback(
|
|||
"""Street-level direct-EPC fallback for listings the strict matcher missed.
|
||||
|
||||
~90% of scraped listings publish a street-level address only ("Oldstead
|
||||
Road, Bromley" — Rightmove never exposes the house number or UPRN), so the
|
||||
Road, Bromley", since Rightmove never exposes the house number or UPRN), so the
|
||||
strict matcher in `_best_listing_match` can never match them against the
|
||||
virtually-always-numbered EPC register and their EPC-derived fields
|
||||
(energy rating, interior height, former-council-house flag, construction
|
||||
|
|
@ -1371,7 +1378,7 @@ def _best_street_epc_fallback(
|
|||
same-postcode-unit preference and a house-number-overlap bonus (a
|
||||
numbered listing that failed the strict set-equality gate, e.g. a
|
||||
"751 753" range vs "751", still lands on the right property). The result
|
||||
is street-representative rather than property-exact — hence the distinct
|
||||
is street-representative rather than property-exact, hence the distinct
|
||||
"street" method label so downstream consumers can tell the two confidence
|
||||
levels apart. The matched certificate's construction year is kept only when
|
||||
the street is era-homogeneous (see
|
||||
|
|
@ -1439,7 +1446,7 @@ def _best_street_epc_fallback(
|
|||
# bathrooms (the upstream storage.py defect noted in
|
||||
# `_finalize_listings`). It systematically over-counts, so comparing
|
||||
# it to the EPC habitable-room count biases selection toward larger,
|
||||
# typically older certificates — the opposite of a useful signal —
|
||||
# typically older certificates (the opposite of a useful signal),
|
||||
# and there is no clean listing-side habitable-room count to use.
|
||||
if (
|
||||
listing_postcode
|
||||
|
|
@ -1472,9 +1479,9 @@ def _load_listings_for_merge(listings_path: Path, arcgis_path: Path) -> pl.DataF
|
|||
"""Read the listings parquet and prepare it for the wide-frame merge.
|
||||
|
||||
Output is keyed by `_listing_idx` and carries:
|
||||
* `postcode` — canonical (NSPL `pcds`) form, with terminated postcodes
|
||||
* `postcode`: canonical (NSPL `pcds`) form, with terminated postcodes
|
||||
remapped to their nearest active successor;
|
||||
* `pp_address` — the listing's raw register address (used as the
|
||||
* `pp_address`: the listing's raw register address (used as the
|
||||
address half of the fuzzy match);
|
||||
* one `_actual_*` overlay column per `_LISTING_OVERLAY_SOURCES` entry.
|
||||
"""
|
||||
|
|
@ -1684,7 +1691,7 @@ def _listing_match_frame(listings: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
Listings are matched to EPC certificates and properties by UPRN and by
|
||||
fuzzy street address within their (now accurate, detail-page-sourced)
|
||||
postcode — never by coordinate proximity — so no projected easting/northing
|
||||
postcode (never by coordinate proximity), so no projected easting/northing
|
||||
is computed here. `_listing_uprn` flows through from the loaded listings.
|
||||
"""
|
||||
return listings.with_columns(
|
||||
|
|
@ -1763,8 +1770,8 @@ def _index_candidates(
|
|||
The EPC register's UPRN is NOT unique: a single building/parent UPRN fans
|
||||
across many distinct flats (up to 58 distinct (address, postcode) rows in
|
||||
the 2026-06 data; ~9k UPRNs collide, touching ~20k epc_pp rows). Such a
|
||||
UPRN cannot serve as a 1:1 exact-match key — it would mis-link a listing to
|
||||
one arbitrary flat — so any UPRN that resolves to more than one distinct
|
||||
UPRN cannot serve as a 1:1 exact-match key (it would mis-link a listing to
|
||||
one arbitrary flat), so any UPRN that resolves to more than one distinct
|
||||
``(postcode_key, address_key)`` identity is dropped from ``uprn_index``;
|
||||
those listings fall back to the fuzzy street-address matcher, which
|
||||
disambiguates the specific flat. A UPRN repeated for the SAME identity
|
||||
|
|
@ -1878,7 +1885,7 @@ def _index_epc_streets(
|
|||
maps outcode -> the tokens appearing in at least a quarter of that
|
||||
outcode's street keys. Those are locality suffixes (LONDON, SURREY, the
|
||||
town name) rather than street names, and a fallback match must be anchored
|
||||
by at least one token that is NOT one of them — otherwise a town-only
|
||||
by at least one token that is NOT one of them. Otherwise a town-only
|
||||
listing address ("COULSDON SURREY") token_set-inflates to 100 against any
|
||||
street key carrying the same locality suffix and matches an arbitrary
|
||||
street in the outcode.
|
||||
|
|
@ -2260,7 +2267,7 @@ def _finalize_listings(df: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
@dataclass
|
||||
class _BuildResult:
|
||||
"""Outputs of `_build` — exactly one of the two slot pairs is populated."""
|
||||
"""Outputs of `_build`: exactly one of the two slot pairs is populated."""
|
||||
|
||||
postcode: pl.DataFrame | None = None
|
||||
properties: pl.DataFrame | None = None
|
||||
|
|
@ -2286,6 +2293,38 @@ def _fill_property_level_no_defaults(frame: pl.LazyFrame) -> pl.LazyFrame:
|
|||
)
|
||||
|
||||
|
||||
def _epc_council_by_postcode(wide: pl.LazyFrame) -> pl.LazyFrame:
|
||||
"""Aggregate the per-property EPC social-tenure flags to POSTCODE percentages.
|
||||
|
||||
Two EPC-derived AREA columns, each a share of *all* deduped dwellings in the
|
||||
postcode (one row per dwelling in ``wide``). This denominator is the dwelling
|
||||
universe, NOT Census households: a dwelling with no EPC is ``was_council_house
|
||||
== "No"`` and so counts against the share, making these EPC-coverage-limited
|
||||
lower bounds that are not directly comparable to the Census ``% Social rent``
|
||||
(which stays at LSOA grain). A postcode holds relatively few dwellings, so
|
||||
these shares are coarse and noisier than an LSOA average.
|
||||
|
||||
* ``% Council housing``: dwellings ever recorded as council/social housing
|
||||
per EPC (``was_council_house == "Yes"``).
|
||||
* ``% Ex-council``: ever-council dwellings whose LATEST EPC certificate is no
|
||||
longer social rented (sold off / no longer social):
|
||||
``was_council_house == "Yes" AND latest_tenure_status != "Rented (social)"``.
|
||||
A null ``latest_tenure_status`` counts as not-currently-social.
|
||||
|
||||
``was_council_house`` is already "Yes"/"No" filled for every row (see
|
||||
``_fill_property_level_no_defaults``), so the means are over the full postcode.
|
||||
Returns a postcode-keyed LazyFrame to left-join onto the AREA frame only.
|
||||
"""
|
||||
currently_social = (pl.col("latest_tenure_status") == "Rented (social)").fill_null(
|
||||
False
|
||||
)
|
||||
ever_social = pl.col("was_council_house") == "Yes"
|
||||
return wide.group_by("postcode").agg(
|
||||
(ever_social.mean() * 100).round(1).alias("% Council housing"),
|
||||
((ever_social & ~currently_social).mean() * 100).round(1).alias("% Ex-council"),
|
||||
)
|
||||
|
||||
|
||||
def _build(
|
||||
epc_pp_path: Path,
|
||||
arcgis_path: Path,
|
||||
|
|
@ -2311,9 +2350,9 @@ def _build(
|
|||
"""Build postcode/properties dataframes (or enriched listings) from epc_pp + auxiliary data.
|
||||
|
||||
Modes:
|
||||
* `normal` — produces (postcode_df, properties_df) as before. Ignores
|
||||
* `normal`: produces (postcode_df, properties_df) as before. Ignores
|
||||
`actual_listings_path` if supplied.
|
||||
* `listings` — requires `actual_listings_path`; produces a single
|
||||
* `listings`: requires `actual_listings_path`; produces a single
|
||||
enriched-listings DataFrame and skips the postcode/properties outputs.
|
||||
Listings flow through the same enrichment joins as historical rows,
|
||||
so postcode-scoped features (tree density, crime, deprivation, …) end
|
||||
|
|
@ -2331,8 +2370,8 @@ def _build(
|
|||
)
|
||||
_validate_lad_source_coverage(iod_path, rental_prices_path)
|
||||
|
||||
# The dwelling universe — floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes — is shared with
|
||||
# The dwelling universe (floor filter, terminated-postcode remap,
|
||||
# collapse-dedupe, restrict to active English postcodes) is shared with
|
||||
# price estimation so estimates line up 1:1 with these rows. See
|
||||
# pipeline.transform.property_base.
|
||||
wide = build_property_base(epc_pp_path, arcgis_path)
|
||||
|
|
@ -2460,6 +2499,17 @@ def _build(
|
|||
wide = _join_area_side_tables(wide, **area_side_tables)
|
||||
postcode_area = _join_area_side_tables(postcode_area, **area_side_tables)
|
||||
|
||||
# EPC-derived council/ex-council shares: aggregate the per-property social
|
||||
# tenure flags to POSTCODE percentages and attach to the AREA frame only
|
||||
# (these are area columns, like the Census tenure block, not per-property).
|
||||
# Built before dropping latest_tenure_status, which is its only consumer.
|
||||
epc_council_by_postcode = _epc_council_by_postcode(wide)
|
||||
postcode_area = postcode_area.join(epc_council_by_postcode, on="postcode", how="left")
|
||||
# latest_tenure_status is property-grain and not in _AREA_COLUMNS, so the
|
||||
# split would otherwise leak it into properties.parquet. It has served its
|
||||
# purpose (the postcode aggregate above), so drop it from the property frame.
|
||||
wide = wide.drop("latest_tenure_status", strict=False)
|
||||
|
||||
# Derive bedroom count: habitable rooms - 1 (assuming 1 reception room), clipped to 0..4
|
||||
wide = wide.with_columns(
|
||||
(pl.col("number_habitable_rooms") - 1)
|
||||
|
|
@ -2649,7 +2699,7 @@ def main():
|
|||
required=False,
|
||||
help=(
|
||||
"Optional scraped-listings parquet. When provided, listings flow "
|
||||
"through the same merge pipeline as historical properties — set "
|
||||
"through the same merge pipeline as historical properties. Set "
|
||||
"--output-listings to write the enriched-listings file instead "
|
||||
"of the postcode/properties files."
|
||||
),
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ GROCERY_STATIC_EXCLUDED_CATEGORIES = {
|
|||
# Scope: "Public Park Or Garden" is the core park function. "Playing Field"
|
||||
# (open public recreation grounds) is borderline but kept: outside big cities
|
||||
# the local rec ground is the de facto park. "Play Space" (playgrounds) is
|
||||
# excluded — a playground is not a park, and "Playground" is already its own
|
||||
# excluded: a playground is not a park, and "Playground" is already its own
|
||||
# OSM-derived category. The remaining functions (Religious Grounds, Golf
|
||||
# Course, Cemetery, Allotments, Bowling Green, Tennis Court, Other Sports
|
||||
# Facility) are clearly not parks.
|
||||
|
|
@ -76,7 +76,7 @@ def _groceries_categories(pois: pl.DataFrame) -> list[str]:
|
|||
with group "Groceries"; it never emits the literal "Supermarket". Collecting
|
||||
every Groceries category captures both the OSM strings and the brand names.
|
||||
Speciality food retail (bakeries, butchers, delis, off-licences) is
|
||||
excluded — see GROCERY_STATIC_EXCLUDED_CATEGORIES.
|
||||
excluded. See GROCERY_STATIC_EXCLUDED_CATEGORIES.
|
||||
"""
|
||||
if "group" not in pois.columns:
|
||||
raise ValueError("POI dataframe must include a 'group' column")
|
||||
|
|
@ -138,7 +138,7 @@ def _greenspace_count_frame(greenspace: pl.DataFrame) -> pl.DataFrame:
|
|||
|
||||
os_greenspace.parquet is one row per ACCESS POINT (park gate), which is the
|
||||
right grain for nearest-distance (the nearest gate is what matters) but
|
||||
wildly over-counts "Number of amenities (Park) within Xkm" — a large park
|
||||
wildly over-counts "Number of amenities (Park) within Xkm": a large park
|
||||
with 30 gates counted as 30 parks. Counting uses one row per site at the
|
||||
site centroid (falling back to the first access point when no centroid is
|
||||
available). Degrades gracefully: a legacy parquet without `site_id` is
|
||||
|
|
|
|||
|
|
@ -125,12 +125,7 @@ async fn static_cache_headers(
|
|||
response
|
||||
}
|
||||
|
||||
/// Add baseline security headers to every response. These are deliberately the
|
||||
/// low-risk, broadly-compatible set: a Content-Security-Policy and
|
||||
/// Permissions-Policy are intentionally left out because this app loads many
|
||||
/// cross-origin resources (maplibre/deck.gl, Stripe, Google Street View,
|
||||
/// analytics, the error sink) and a mis-scoped policy would break the map or
|
||||
/// checkout. They should be added later as report-only first.
|
||||
|
||||
async fn security_headers(
|
||||
request: axum::extract::Request,
|
||||
next: middleware::Next,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, LazyLock, Mutex};
|
||||
|
||||
use axum::extract::rejection::JsonRejection;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
|
|
@ -153,7 +154,7 @@ fn current_unix_secs_string() -> String {
|
|||
|
||||
/// Fetch the live `is_admin` flag for a user, bypassing any cached token
|
||||
/// claims. Returns Err with an HTTP response if PocketBase is unreachable
|
||||
/// or returns an unexpected payload — the caller should propagate that.
|
||||
/// or returns an unexpected payload. The caller should propagate that.
|
||||
async fn verify_is_admin(
|
||||
state: &AppState,
|
||||
pb_url: &str,
|
||||
|
|
@ -295,7 +296,7 @@ async fn mark_invite_used(
|
|||
// Defense in depth: PocketBase has no atomic compare-and-swap for record
|
||||
// updates, and our local + distributed locks could in principle fail (lock
|
||||
// server timeout, server restart mid-redemption). Re-read the record and
|
||||
// confirm WE actually own it — if a concurrent redemption beat us to the
|
||||
// confirm WE actually own it. If a concurrent redemption beat us to the
|
||||
// PATCH, both writes succeeded but the loser's user_id is overwritten and
|
||||
// we must NOT grant a license.
|
||||
let verify_url = format!("{pb_url}/api/collections/invites/records/{invite_id}");
|
||||
|
|
@ -328,7 +329,7 @@ async fn mark_invite_used(
|
|||
invite_id,
|
||||
expected = user_id,
|
||||
actual = actual_user,
|
||||
"Invite redemption race lost — invite already claimed by another user"
|
||||
"Invite redemption race lost. Invite already claimed by another user"
|
||||
);
|
||||
return Err((StatusCode::CONFLICT, "Invite was already redeemed").into_response());
|
||||
}
|
||||
|
|
@ -472,7 +473,7 @@ pub async fn post_invites(
|
|||
/// Only recognized when `--dist` is not set (i.e., dev mode).
|
||||
const DEV_INVITE_CODE: &str = "devdevdevdev";
|
||||
|
||||
/// Validate an invite code. Public endpoint — codes are 12-char random alphanumeric
|
||||
/// Validate an invite code. Public endpoint: codes are 12-char random alphanumeric
|
||||
/// so enumeration is impractical, and the response only reveals valid/invalid + type.
|
||||
pub async fn get_invite(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
|
|
@ -542,7 +543,7 @@ pub async fn get_invite(
|
|||
let used = !used_by.is_empty();
|
||||
let created_by = invite["created_by"].as_str().unwrap_or("");
|
||||
|
||||
// Look up inviter's name (email local part) — sanitized before returning.
|
||||
// Look up inviter's name (email local part), sanitized before returning.
|
||||
let invited_by = if !created_by.is_empty() {
|
||||
let user_url = format!("{pb_url}/api/collections/users/records/{created_by}");
|
||||
match state
|
||||
|
|
@ -594,7 +595,7 @@ pub async fn get_invite(
|
|||
pub async fn post_redeem_invite(
|
||||
State(shared): State<Arc<SharedState>>,
|
||||
Extension(user): Extension<OptionalUser>,
|
||||
Json(req): Json<RedeemRequest>,
|
||||
body: Result<Json<RedeemRequest>, JsonRejection>,
|
||||
) -> Response {
|
||||
let state = shared.load_state();
|
||||
let user = match user.0 {
|
||||
|
|
@ -602,11 +603,20 @@ pub async fn post_redeem_invite(
|
|||
None => return StatusCode::UNAUTHORIZED.into_response(),
|
||||
};
|
||||
|
||||
// Parse the body only after authenticating, and return a clean message
|
||||
// instead of leaking the raw serde rejection (e.g. "missing field `code`").
|
||||
let req = match body {
|
||||
Ok(Json(req)) => req,
|
||||
Err(_) => {
|
||||
return (StatusCode::BAD_REQUEST, "A valid invite code is required.").into_response()
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(msg) = validate_invite_code(&req.code) {
|
||||
return (StatusCode::BAD_REQUEST, msg).into_response();
|
||||
}
|
||||
|
||||
// Dev-only: fake redeem — just return "licensed" without touching PocketBase
|
||||
// Dev-only: fake redeem, just return "licensed" without touching PocketBase
|
||||
if state.is_dev && req.code == DEV_INVITE_CODE {
|
||||
info!(user_id = %user.id, "Dev invite redeemed (no-op)");
|
||||
return Json(RedeemResponse {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue