Lots of improvements

This commit is contained in:
Andras Schmelczer 2026-03-10 22:05:51 +00:00
parent ef921361ec
commit 80a5a2a774
21 changed files with 489 additions and 337 deletions

View file

@ -20,13 +20,6 @@ const ACCOUNT_TABS = [
{ key: 'settings', label: 'Settings' },
];
const SUBSCRIPTION_OPTIONS = ['free', 'licensed'] as const;
const SUBSCRIPTION_LABELS: Record<string, string> = {
free: 'Free',
licensed: 'Licensed',
};
function SavedSearchesContent({
searches,
loading,
@ -196,10 +189,6 @@ function SettingsContent({
onRefreshAuth: () => Promise<void>;
onRequestVerification: (email: string) => Promise<void>;
}) {
const [selectedSubscription, setSelectedSubscription] = useState(user.subscription || 'free');
const [saving, setSaving] = useState(false);
const [saved, setSaved] = useState(false);
const [error, setError] = useState<string | null>(null);
const [newsletterSaving, setNewsletterSaving] = useState(false);
const [newsletterError, setNewsletterError] = useState<string | null>(null);
@ -207,65 +196,50 @@ function SettingsContent({
const [verificationSending, setVerificationSending] = useState(false);
const [verificationSent, setVerificationSent] = useState(false);
// Invite state
const [creatingInvite, setCreatingInvite] = useState(false);
const [inviteUrl, setInviteUrl] = useState<string | null>(null);
const [inviteError, setInviteError] = useState<string | null>(null);
const [inviteCopied, setInviteCopied] = useState(false);
// Invite state — keyed by invite type for admins (who can create both kinds)
const [creatingInvite, setCreatingInvite] = useState<Record<string, boolean>>({});
const [inviteUrl, setInviteUrl] = useState<Record<string, string>>({});
const [inviteError, setInviteError] = useState<Record<string, string>>({});
const [inviteCopied, setInviteCopied] = useState<Record<string, boolean>>({});
const handleSave = async () => {
setSaving(true);
setError(null);
setSaved(false);
try {
const res = await fetch(apiUrl('subscription'), {
method: 'PATCH',
...authHeaders({
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ subscription: selectedSubscription }),
}),
});
assertOk(res, 'Update subscription');
await onRefreshAuth();
setSaved(true);
setTimeout(() => setSaved(false), 2000);
} catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to update subscription';
setError(msg);
} finally {
setSaving(false);
}
};
const handleCreateInvite = async () => {
setCreatingInvite(true);
setInviteError(null);
setInviteUrl(null);
setInviteCopied(false);
const handleCreateInvite = async (type: string) => {
setCreatingInvite((prev) => ({ ...prev, [type]: true }));
setInviteError((prev) => {
const next = { ...prev };
delete next[type];
return next;
});
setInviteUrl((prev) => {
const next = { ...prev };
delete next[type];
return next;
});
setInviteCopied((prev) => ({ ...prev, [type]: false }));
try {
const res = await fetch(apiUrl('invites'), {
method: 'POST',
...authHeaders({
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({}),
body: JSON.stringify({ invite_type: type }),
}),
});
assertOk(res, 'Create invite');
const data = await res.json();
setInviteUrl(data.url);
setInviteUrl((prev) => ({ ...prev, [type]: data.url }));
} catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to create invite';
setInviteError(msg);
setInviteError((prev) => ({ ...prev, [type]: msg }));
} finally {
setCreatingInvite(false);
setCreatingInvite((prev) => ({ ...prev, [type]: false }));
}
};
const handleCopyInvite = () => {
if (!inviteUrl) return;
copyToClipboard(inviteUrl, () => {
setInviteCopied(true);
setTimeout(() => setInviteCopied(false), 2000);
const handleCopyInvite = (type: string) => {
const url = inviteUrl[type];
if (!url) return;
copyToClipboard(url, () => {
setInviteCopied((prev) => ({ ...prev, [type]: true }));
setTimeout(() => setInviteCopied((prev) => ({ ...prev, [type]: false })), 2000);
});
};
@ -323,7 +297,7 @@ function SettingsContent({
<div>
<p className="text-sm text-warm-500 dark:text-warm-400">Subscription</p>
<span className={`inline-block text-sm font-medium px-2.5 py-0.5 rounded-full mt-1 ${badgeColor}`}>
{SUBSCRIPTION_LABELS[user.subscription] || user.subscription || 'Free'}
{user.subscription === 'licensed' ? 'Licensed' : 'Free'}
</span>
</div>
</div>
@ -369,83 +343,48 @@ function SettingsContent({
</div>
{/* Invite friends */}
{isLicensed && (
<div className="px-5 py-4">
<p className="text-sm text-warm-500 dark:text-warm-400 mb-3">
{user.isAdmin ? 'Invite friends (100% off)' : 'Invite friends (30% off)'}
</p>
{inviteUrl ? (
<div className="flex items-center gap-2">
<input
type="text"
readOnly
value={inviteUrl}
className="flex-1 px-3 py-2 rounded-lg border border-warm-200 dark:border-warm-700 bg-warm-50 dark:bg-warm-900 text-navy-950 dark:text-warm-200 text-sm"
/>
{isLicensed &&
(user.isAdmin ? ['admin', 'referral'] : ['referral']).map((type) => (
<div key={type} className="px-5 py-4">
<p className="text-sm text-warm-500 dark:text-warm-400 mb-3">
{type === 'admin' ? 'Invite friends (100% off)' : 'Invite friends (30% off)'}
</p>
{inviteUrl[type] ? (
<div className="flex items-center gap-2">
<input
type="text"
readOnly
value={inviteUrl[type]}
className="flex-1 px-3 py-2 rounded-lg border border-warm-200 dark:border-warm-700 bg-warm-50 dark:bg-warm-900 text-navy-950 dark:text-warm-200 text-sm"
/>
<button
onClick={() => handleCopyInvite(type)}
className="px-3 py-2 rounded-lg bg-teal-600 hover:bg-teal-700 text-white text-sm font-medium flex items-center gap-1.5"
>
{inviteCopied[type] ? (
<CheckIcon className="w-4 h-4" />
) : (
<ClipboardIcon className="w-4 h-4" />
)}
{inviteCopied[type] ? 'Copied' : 'Copy'}
</button>
</div>
) : (
<button
onClick={handleCopyInvite}
className="px-3 py-2 rounded-lg bg-teal-600 hover:bg-teal-700 text-white text-sm font-medium flex items-center gap-1.5"
onClick={() => handleCreateInvite(type)}
disabled={!!creatingInvite[type]}
className="px-4 py-2 rounded-lg bg-teal-600 hover:bg-teal-700 text-white text-sm font-medium disabled:opacity-50 disabled:cursor-wait flex items-center gap-2"
>
{inviteCopied ? (
<CheckIcon className="w-4 h-4" />
) : (
<ClipboardIcon className="w-4 h-4" />
)}
{inviteCopied ? 'Copied' : 'Copy'}
{creatingInvite[type] && <SpinnerIcon className="w-4 h-4 animate-spin" />}
{type === 'admin' ? 'Generate free invite link' : 'Generate referral link'}
</button>
</div>
) : (
<button
onClick={handleCreateInvite}
disabled={creatingInvite}
className="px-4 py-2 rounded-lg bg-teal-600 hover:bg-teal-700 text-white text-sm font-medium disabled:opacity-50 disabled:cursor-wait flex items-center gap-2"
>
{creatingInvite && <SpinnerIcon className="w-4 h-4 animate-spin" />}
{user.isAdmin ? 'Generate free invite link' : 'Generate referral link'}
</button>
)}
{inviteError && (
<p className="mt-2 text-sm text-red-600 dark:text-red-400">{inviteError}</p>
)}
</div>
)}
{/* Admin section */}
{user.isAdmin && (
<div className="px-5 py-4">
<p className="text-sm text-warm-500 dark:text-warm-400 mb-3">
Admin: Change subscription
</p>
<div className="flex items-center gap-3">
<select
value={selectedSubscription}
onChange={(e) => setSelectedSubscription(e.target.value)}
className="flex-1 px-3 py-2 rounded-lg border border-warm-200 dark:border-warm-700 bg-white dark:bg-warm-900 text-navy-950 dark:text-warm-200 text-sm"
>
{SUBSCRIPTION_OPTIONS.map((opt) => (
<option key={opt} value={opt}>
{SUBSCRIPTION_LABELS[opt]}
</option>
))}
</select>
<button
onClick={handleSave}
disabled={saving || selectedSubscription === user.subscription}
className="px-4 py-2 rounded-lg bg-teal-600 hover:bg-teal-700 text-white text-sm font-medium disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2"
>
{saving ? (
<SpinnerIcon className="w-4 h-4 animate-spin" />
) : saved ? (
<CheckIcon className="w-4 h-4" />
) : null}
{saved ? 'Saved' : 'Save'}
</button>
)}
{inviteError[type] && (
<p className="mt-2 text-sm text-red-600 dark:text-red-400">{inviteError[type]}</p>
)}
</div>
{error && (
<p className="mt-2 text-sm text-red-600 dark:text-red-400">{error}</p>
)}
</div>
)}
))}
</div>
{/* Support */}

View file

@ -48,7 +48,7 @@ const DATA_SOURCES = [
id: 'ethnicity',
name: 'Population by Ethnicity (2021 Census)',
origin: 'ONS',
use: 'Population percentages by ethnic group (Asian, Black, Mixed, White, Other) per Local Authority. Joined via Local Authority District code.',
use: 'Population percentages by ethnic group (South Asian, East Asian, Black, Mixed, White, Other) per Local Authority. Joined via Local Authority District code.',
url: 'https://www.ethnicity-facts-figures.service.gov.uk/uk-population-by-ethnicity/national-and-regional-populations/regional-ethnic-diversity/latest/#download-the-data',
license: 'Open Government Licence v3.0',
},
@ -60,14 +60,6 @@ const DATA_SOURCES = [
url: 'https://data.police.uk/data/',
license: 'Open Government Licence v3.0',
},
{
id: 'tfl-journey-times',
name: 'TfL Journey Times',
origin: 'Transport for London',
use: "Journey time calculations from postcodes to central London destinations (Bank, Waterloo, King's Cross, etc.) via public transport and cycling.",
url: 'https://api-portal.tfl.gov.uk/',
license: 'Powered by TfL Open Data',
},
{
id: 'osm-pois',
name: 'OpenStreetMap POIs',

View file

@ -2,6 +2,12 @@ import type { FeatureMeta } from '../../types';
import { InfoIcon } from './icons';
import { getGroupIcon } from '../../lib/group-icons';
const MODE_LABELS: Record<string, string> = {
historical: 'Historical',
buy: 'Buy',
rent: 'Rent',
};
interface FeatureLabelProps {
feature: FeatureMeta;
onShowInfo?: (feature: FeatureMeta) => void;
@ -17,6 +23,10 @@ export function FeatureLabel({
}: FeatureLabelProps) {
const textClass = size === 'sm' ? 'text-sm' : 'text-xs';
const GroupIcon = feature.group ? getGroupIcon(feature.group) : null;
const modeTag =
feature.modes && feature.modes.length > 0
? feature.modes.map((m) => MODE_LABELS[m] || m).join(' · ')
: null;
return (
<div
@ -30,6 +40,11 @@ export function FeatureLabel({
>
{feature.name}
</span>
{modeTag && (
<span className="shrink-0 text-[10px] leading-none font-medium px-1.5 py-0.5 rounded-full bg-warm-100 dark:bg-warm-800 text-warm-500 dark:text-warm-400 border border-warm-200 dark:border-warm-700">
{modeTag}
</span>
)}
{feature.detail && onShowInfo && (
<button
onClick={() => onShowInfo(feature)}

View file

@ -20,10 +20,10 @@ export default function InfoPopup({ title, children, onClose, sourceLink }: Info
useClickOutside(popupRef, handleClose);
return (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/30">
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/30 p-4">
<div
ref={popupRef}
className="bg-white dark:bg-navy-800 border border-warm-200 dark:border-navy-700 rounded-lg shadow-xl max-w-md w-full mx-4 p-5"
className="bg-white dark:bg-navy-800 border border-warm-200 dark:border-navy-700 rounded-lg shadow-xl max-w-md w-full max-h-full overflow-y-auto p-5"
>
<div className="flex items-start justify-between mb-3">
<h3 className="text-sm font-semibold text-warm-900 dark:text-warm-100 pr-4">{title}</h3>

View file

@ -169,6 +169,9 @@ export function useMapData({
const params = new URLSearchParams({ bounds: boundsStr });
if (filtersStr) params.set('filters', filtersStr);
params.set('fields', viewFeature && !viewFeature.startsWith('tt_') ? viewFeature : '');
if (travelParam) {
params.set('travel', travelParam);
}
const res = await fetch(
apiUrl('postcodes', params),
authHeaders({
@ -261,7 +264,7 @@ export function useMapData({
const vals: number[] = [];
if (usePostcodeView && !isTravelTime) {
if (usePostcodeView) {
if (effectivePostcodeData.length === 0) return null;
for (const feat of effectivePostcodeData) {
if (bounds) {

View file

@ -17,6 +17,10 @@ export function useTheme() {
root.classList.remove('dark');
}
localStorage.setItem('theme', theme);
const color = theme === 'dark' ? '#0a0e1a' : '#fafaf9';
document.querySelectorAll('meta[name="theme-color"]').forEach((m) => {
m.setAttribute('content', color);
});
}, [theme]);
const toggleTheme = useCallback(() => {

View file

@ -13,6 +13,11 @@ body,
overscroll-behavior: none;
}
html {
background-color: #fafaf9;
color-scheme: light;
}
html.dark {
background-color: #0a0e1a;
color-scheme: dark;

View file

@ -119,7 +119,7 @@ export const STACKED_GROUPS: Record<
{
label: 'Ethnic composition',
unit: '%',
components: ['% White', '% Asian', '% Black', '% Mixed', '% Other'],
components: ['% White', '% South Asian', '% East Asian', '% Black', '% Mixed', '% Other'],
},
],
};
@ -153,8 +153,8 @@ export const STACKED_ENUM_GROUPS: Record<
},
{
label: 'Leasehold/Freehold',
feature: 'Leashold/Freehold',
components: ['Leashold/Freehold'],
feature: 'Leasehold/Freehold',
components: ['Leasehold/Freehold'],
valueOrder: ['Freehold', 'Leasehold'],
valueColors: ['#3b82f6', '#f59e0b'],
},

View file

@ -113,7 +113,7 @@ export function buildPropertySearchUrls({
const maxBathrooms =
Array.isArray(bathroomFilter) && typeof bathroomFilter[1] === 'number' ? bathroomFilter[1] : undefined;
const tenureFilter = filters['Leashold/Freehold'];
const tenureFilter = filters['Leasehold/Freehold'];
const selectedTenures =
Array.isArray(tenureFilter) && typeof tenureFilter[0] === 'string'
? (tenureFilter as string[])

View file

@ -15,7 +15,8 @@ export function formatValue(value: number, fmt?: ValueFormat): string {
return `${p}${value.toFixed(1)}${s}`;
}
export function formatFilterValue(value: number): string {
export function formatFilterValue(value: number, raw?: boolean): string {
if (raw) return Math.round(value).toString();
if (Math.abs(value) >= 1_000_000) return `${(value / 1_000_000).toFixed(1)}M`;
if (Math.abs(value) >= 1_000) return `${(value / 1_000).toFixed(1)}k`;
if (Number.isInteger(value)) return value.toString();

File diff suppressed because one or more lines are too long

View file

@ -112,13 +112,19 @@ export interface PlaceResult {
city?: string;
}
export interface JourneyLeg {
mode: string;
from?: string;
to?: string;
minutes: number;
}
export interface RenovationEvent {
year: number;
event: string;
}
export interface Property {
// String fields
address?: string;
postcode?: string;
property_type?: string;

View file

@ -115,15 +115,17 @@ class PlaceHandler(osmium.SimpleHandler):
self._add(name, place_type, lat, lon, population)
return
# Tube stations only (London Underground)
# Railway stations (tube, national rail, DLR, overground, Elizabeth line)
if n.tags.get("railway") == "station":
tags = dict(n.tags)
station_tag = tags.get("station", "")
network = tags.get("network", "").lower()
if station_tag == "subway" or "underground" in network:
display_name = _station_display_name(name, tags)
self._add(display_name, "station", lat, lon, population)
# Skip tram stops
if station_tag == "light_rail" or "tramlink" in network or "tram" in network:
return
display_name = _station_display_name(name, tags)
self._add(display_name, "station", lat, lon, population)
return
def main() -> None:
@ -137,7 +139,7 @@ def main() -> None:
args = parser.parse_args()
pbf_file = args.pbf
print("Extracting place nodes: cities + tube stations")
print("Extracting place nodes: cities + railway stations")
with tqdm(
unit=" elements",
unit_scale=True,

View file

@ -0,0 +1,74 @@
"""Fetch Rightmove outcode→ID mapping for all outcodes in postcode.parquet."""
import argparse
import json
from pathlib import Path
import httpx
import polars as pl
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
def fetch_outcode_ids(postcodes_path: Path, output: Path) -> None:
df = pl.read_parquet(postcodes_path, columns=["Postcode"])
outcodes = sorted(
set(df["Postcode"].str.split(" ").list.first().to_list()) - {""}
)
print(f"Querying Rightmove typeahead for {len(outcodes)} outcodes...")
mapping: dict[str, str] = {}
missed: list[str] = []
client = httpx.Client(timeout=10)
for i, oc in enumerate(outcodes):
try:
resp = client.get(TYPEAHEAD_URL, params={"query": oc, "limit": "5"})
data = resp.json()
found = False
for m in data.get("matches", []):
if (
m["type"] == "OUTCODE"
and m["displayName"].upper().replace(" ", "")
== oc.upper().replace(" ", "")
):
mapping[oc] = str(m["id"])
found = True
break
if not found:
missed.append(oc)
except Exception as e:
missed.append(oc)
print(f" Error for {oc}: {e}")
if (i + 1) % 200 == 0:
print(f" {i + 1}/{len(outcodes)} done ({len(mapping)} found)")
client.close()
output.parent.mkdir(parents=True, exist_ok=True)
with open(output, "w") as f:
json.dump(mapping, f, sort_keys=True)
print(f"Wrote {output} ({len(mapping)} outcodes, {len(missed)} missed)")
if missed:
print(f"Missed: {missed}")
def main() -> None:
parser = argparse.ArgumentParser(
description="Fetch Rightmove outcode ID mapping"
)
parser.add_argument(
"--postcodes", type=Path, required=True, help="postcode.parquet path"
)
parser.add_argument(
"--output", type=Path, required=True, help="Output JSON file path"
)
args = parser.parse_args()
fetch_outcode_ids(args.postcodes, args.output)
if __name__ == "__main__":
main()

View file

@ -8,6 +8,7 @@ Downloads:
Then processes for R5 compatibility:
- Cleans BODS GTFS (fixes stop_times >72h, feed_info year >2100)
- Converts high-frequency metro/tram services to frequency-based GTFS
- Converts TfL TransXChange to GTFS via transxchange2gtfs
- Converts National Rail CIF to GTFS via dtd2mysql (requires MariaDB Docker)
@ -20,12 +21,15 @@ Output directory: property-data/transit/
import argparse
import json
import os
import shutil
import statistics
import subprocess
import tempfile
import time
import urllib.parse
import urllib.request
import zipfile
from collections import defaultdict
from pathlib import Path
from tqdm import tqdm
@ -184,6 +188,229 @@ def clean_gtfs(src: Path, dst: Path) -> None:
print(f" Saved to {dst}")
def _parse_gtfs_time(time_str: str) -> int | None:
"""Parse HH:MM:SS to seconds since midnight. Returns None on failure."""
time_str = time_str.strip('"')
if ":" not in time_str:
return None
try:
h, m, s = time_str.split(":")
return int(h) * 3600 + int(m) * 60 + int(s)
except ValueError:
return None
def _secs_to_gtfs_time(s: int) -> str:
"""Convert seconds since midnight to HH:MM:SS."""
h = s // 3600
m = (s % 3600) // 60
sec = s % 60
return f"{h:02d}:{m:02d}:{sec:02d}"
def convert_high_freq_to_frequency_based(
src: Path, dst: Path, *, max_headway_minutes: int = 15
) -> None:
"""Convert high-frequency scheduled services to frequency-based GTFS entries.
Identifies metro (route_type=1) and tram (route_type=0) routes with regular
headways under max_headway_minutes, then creates frequencies.txt entries and
removes redundant trips. R5's RAPTOR produces smoother percentile results for
frequency-based services, matching the "just turn up" reality of high-frequency
metro/tram services.
"""
if dst.exists():
print(f"Frequency-converted GTFS already exists: {dst}")
return
print("Converting high-frequency services to frequency-based...")
max_headway_secs = max_headway_minutes * 60
with zipfile.ZipFile(src, "r") as zin:
# Step 1: Find metro/tram route IDs
target_route_ids: set[str] = set()
with zin.open("routes.txt") as f:
header = f.readline().decode("utf-8").strip()
cols = header.split(",")
route_id_idx = cols.index("route_id")
rt_idx = cols.index("route_type")
for line in f:
parts = line.decode("utf-8", errors="replace").strip().split(",")
if not parts:
continue
route_type = parts[rt_idx].strip('"')
if route_type in ("0", "1"): # tram, metro/subway
target_route_ids.add(parts[route_id_idx].strip('"'))
if not target_route_ids:
print(" No metro/tram routes found, copying unchanged")
shutil.copy2(src, dst)
return
print(f" Found {len(target_route_ids)} metro/tram routes")
# Step 2: Map target trips to grouping keys
trip_group_key: dict[str, tuple[str, str, str]] = {}
with zin.open("trips.txt") as f:
header = f.readline().decode("utf-8").strip()
cols = header.split(",")
trip_id_idx = cols.index("trip_id")
route_id_idx = cols.index("route_id")
dir_idx = cols.index("direction_id") if "direction_id" in cols else -1
service_idx = cols.index("service_id")
for line in f:
parts = line.decode("utf-8", errors="replace").strip().split(",")
if not parts:
continue
route_id = parts[route_id_idx].strip('"')
if route_id in target_route_ids:
trip_id = parts[trip_id_idx].strip('"')
direction = parts[dir_idx].strip('"') if dir_idx >= 0 else "0"
service_id = parts[service_idx].strip('"')
trip_group_key[trip_id] = (route_id, direction, service_id)
print(f" Found {len(trip_group_key)} trips on target routes")
# Step 3: Get first departure time and first stop for each target trip
trip_first_dep: dict[str, int] = {}
trip_first_stop: dict[str, str] = {}
with zin.open("stop_times.txt") as f:
header = f.readline().decode("utf-8").strip()
cols = header.split(",")
trip_id_idx = cols.index("trip_id")
dep_idx = cols.index("departure_time")
seq_idx = cols.index("stop_sequence")
stop_id_idx = cols.index("stop_id")
for line in f:
parts = line.decode("utf-8", errors="replace").strip().split(",")
if not parts:
continue
trip_id = parts[trip_id_idx].strip('"')
if trip_id not in trip_group_key:
continue
if parts[seq_idx].strip('"') == "0":
dep_secs = _parse_gtfs_time(parts[dep_idx])
if dep_secs is not None:
trip_first_dep[trip_id] = dep_secs
trip_first_stop[trip_id] = parts[stop_id_idx].strip('"')
# Step 4: Group trips by (route, direction, service, first_stop) and compute headways
groups: dict[tuple[str, ...], list[tuple[str, int]]] = defaultdict(list)
for trip_id, dep_secs in trip_first_dep.items():
route_id, direction, service_id = trip_group_key[trip_id]
first_stop = trip_first_stop.get(trip_id, "")
key = (route_id, direction, service_id, first_stop)
groups[key].append((trip_id, dep_secs))
trips_to_remove: set[str] = set()
frequency_entries: list[tuple[str, int, int, int]] = []
groups_converted = 0
for _key, trips in groups.items():
if len(trips) < 4:
continue
trips.sort(key=lambda x: x[1])
headways = [trips[i + 1][1] - trips[i][1] for i in range(len(trips) - 1)]
headways = [h for h in headways if h > 0]
if len(headways) < 3:
continue
median_hw = statistics.median(headways)
if median_hw > max_headway_secs or median_hw < 30:
continue
mean_hw = statistics.mean(headways)
if mean_hw == 0:
continue
stdev_hw = statistics.stdev(headways) if len(headways) > 1 else 0
if stdev_hw / mean_hw > 0.5:
continue
# Convert: keep first trip as template, remove the rest
template_trip_id = trips[0][0]
start_secs = trips[0][1]
end_secs = trips[-1][1] + int(median_hw)
headway_rounded = max(60, round(median_hw / 60) * 60)
frequency_entries.append((template_trip_id, start_secs, end_secs, headway_rounded))
for trip_id, _ in trips[1:]:
trips_to_remove.add(trip_id)
groups_converted += 1
print(f" Converted {groups_converted} trip groups to frequency-based")
print(f" Removing {len(trips_to_remove)} redundant trips")
print(f" Created {len(frequency_entries)} frequency entries")
# Step 5: Write modified GTFS
with zipfile.ZipFile(src, "r") as zin, zipfile.ZipFile(
dst, "w", zipfile.ZIP_DEFLATED
) as zout:
for info in zin.infolist():
if info.filename == "trips.txt":
with zin.open(info) as f:
header = f.readline()
header_str = header.decode("utf-8").strip()
cols = header_str.split(",")
trip_id_idx = cols.index("trip_id")
tmp = tempfile.NamedTemporaryFile(
mode="wb", delete=False, suffix=".txt"
)
tmp.write(header)
for line in f:
parts = (
line.decode("utf-8", errors="replace").strip().split(",")
)
if not parts:
continue
if parts[trip_id_idx].strip('"') not in trips_to_remove:
tmp.write(line)
tmp.close()
zout.write(tmp.name, "trips.txt")
os.unlink(tmp.name)
elif info.filename == "stop_times.txt":
with zin.open(info) as f:
header = f.readline()
header_str = header.decode("utf-8").strip()
cols = header_str.split(",")
trip_id_idx = cols.index("trip_id")
tmp = tempfile.NamedTemporaryFile(
mode="wb", delete=False, suffix=".txt"
)
tmp.write(header)
for line in f:
parts = (
line.decode("utf-8", errors="replace").strip().split(",")
)
if not parts:
continue
if parts[trip_id_idx].strip('"') not in trips_to_remove:
tmp.write(line)
tmp.close()
zout.write(tmp.name, "stop_times.txt")
os.unlink(tmp.name)
elif info.filename == "frequencies.txt":
pass # we'll write our own below
else:
zout.writestr(info, zin.read(info))
# Write frequencies.txt
freq_lines = ["trip_id,start_time,end_time,headway_secs,exact_times\n"]
for trip_id, start, end, headway in frequency_entries:
freq_lines.append(
f"{trip_id},{_secs_to_gtfs_time(start)},{_secs_to_gtfs_time(end)},{headway},0\n"
)
zout.writestr("frequencies.txt", "".join(freq_lines))
print(f" Saved to {dst}")
def download_tfl_transxchange(raw_dir: Path) -> Path:
"""Download TfL TransXChange timetable bundle."""
dest = raw_dir / "tfl_transxchange.zip"
@ -655,12 +882,15 @@ def main() -> None:
raw_dir = output_dir / "raw"
raw_dir.mkdir(parents=True, exist_ok=True)
# 1. Download and clean BODS GTFS
# 1. Download, clean, and frequency-convert BODS GTFS
download_osm_pbf(raw_dir)
bods_raw = download_bods_gtfs(raw_dir)
bods_clean = output_dir / "bods_gtfs.zip"
clean_gtfs(bods_raw, bods_clean)
bods_cleaned = raw_dir / "bods_gtfs_cleaned.zip"
clean_gtfs(bods_raw, bods_cleaned)
bods_final = output_dir / "bods_gtfs.zip"
convert_high_freq_to_frequency_based(bods_cleaned, bods_final)
# 2. TfL TransXChange → GTFS
if args.skip_tfl:

View file

@ -464,7 +464,7 @@ impl PropertyData {
tracing::info!("Concatenating all data sources");
let buy_count = listings_buy.height();
let rent_count = listings_rent.height();
let mut combined = concat(
let combined = concat(
[
properties_joined.lazy(),
listings_buy.lazy(),
@ -495,36 +495,8 @@ impl PropertyData {
let numeric_names = features::all_numeric_feature_names();
let enum_names = features::all_enum_feature_names();
// Fill in NaN/empty placeholder columns for features that don't exist in all
// sources (e.g. Listing date only comes from listings, Estimated current price
// only from properties). Without this, diagonal concat leaves them absent.
{
let schema = combined.schema();
let mut fill_exprs: Vec<Expr> = Vec::new();
for &name in &numeric_names {
if schema.get(name).is_none() {
tracing::info!(feature = %name, "Adding NaN placeholder for missing numeric feature");
fill_exprs.push(lit(f32::NAN).alias(name));
}
}
for &name in &enum_names {
if schema.get(name).is_none() {
tracing::info!(feature = %name, "Adding empty placeholder for missing enum feature");
fill_exprs.push(lit("").alias(name));
}
}
if !fill_exprs.is_empty() {
combined = combined
.lazy()
.with_columns(fill_exprs)
.collect()
.context("Failed to add placeholder columns for missing features")?;
}
}
let schema = combined.schema();
// Validate: every configured feature exists in combined schema
for name in &numeric_names {
match schema.get(name) {
Some(dtype) if is_numeric_dtype(dtype) => {}

View file

@ -92,7 +92,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 10000.0,
description: "Inflation-adjusted estimate of the current property value",
detail: "Estimated by applying a repeat-sales price index to the last known sale price, plus a renovation premium for properties with post-sale improvements detected from EPC records (extensions, renovations, remodeling). The index tracks price changes within each postcode sector and property type. Renovation premiums are estimated per area from observed repeat-sale pairs and decay over time. Properties sold recently will have estimates close to their sale price; older sales are adjusted more.",
detail: "Estimated by applying a repeat-sales price index to the last known sale price, plus a renovation premium for properties with post-sale improvements detected from EPC records (extensions, renovations, remodelling). The index tracks price changes within each postcode sector and property type. Renovation premiums are estimated per area from observed repeat-sale pairs and decay over time. Properties sold recently will have estimates close to their sale price; older sales are adjusted more.",
source: "price-paid",
prefix: "£",
suffix: "",
@ -259,7 +259,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
},
step: 50.0,
description: "Listed monthly rent for properties currently for rent",
detail: "The advertised rental price normalized to monthly for properties currently listed for rent on online property portals. Weekly rents are converted (×52/12), yearly (/12), daily (×365.25/12), and quarterly (/3). Only populated for 'For rent' listings.",
detail: "The advertised rental price normalised to monthly for properties currently listed for rent on online property portals. Weekly rents are converted (×52/12), yearly (/12), daily (×365.25/12), and quarterly (/3). Only populated for 'For rent' listings.",
source: "online-listings",
prefix: "£",
suffix: "/mo",
@ -325,82 +325,14 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
name: "Transport",
features: &[
FeatureConfig {
name: "Public transport to Bank (mins)",
bounds: Bounds::Fixed {
min: 0.0,
max: 180.0,
},
step: 2.0,
description: "Public transport journey time to Bank station",
detail: "Journey time in minutes by public transport to Bank station in the City of London, using TfL's Journey Planner API. Calculated for weekday morning commute times.",
source: "tfl-journey-times",
prefix: "",
suffix: " mins",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
FeatureConfig {
name: "Public transport to Fitzrovia (mins)",
bounds: Bounds::Fixed {
min: 0.0,
max: 180.0,
},
step: 2.0,
description: "Public transport journey time to Fitzrovia",
detail: "Journey time in minutes by public transport to Fitzrovia in central London, using TfL's Journey Planner API. Calculated for weekday morning commute times.",
source: "tfl-journey-times",
prefix: "",
suffix: " mins",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
FeatureConfig {
name: "Cycling to Bank (mins)",
bounds: Bounds::Fixed {
min: 0.0,
max: 180.0,
},
step: 1.0,
description: "Cycling time to Bank station",
detail: "Cycling journey time in minutes to Bank station, as calculated by the TfL Journey Planner API. Uses TfL's default cycling speed and route preferences.",
source: "tfl-journey-times",
prefix: "",
suffix: " mins",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
FeatureConfig {
name: "Cycling to Fitzrovia (mins)",
bounds: Bounds::Fixed {
min: 0.0,
max: 180.0,
},
step: 1.0,
description: "Cycling time to Fitzrovia",
detail: "Cycling journey time in minutes to Fitzrovia, as calculated by the TfL Journey Planner API. Uses TfL's default cycling speed and route preferences.",
source: "tfl-journey-times",
prefix: "",
suffix: " mins",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
FeatureConfig {
name: "Number of public transport stations within 2km",
name: "Train or tube stations within 1km",
bounds: Bounds::Percentile {
low: 5.0,
high: 95.0,
},
step: 1.0,
description: "Number of public transport stops within 2km",
detail: "Count of bus stops, rail stations, tube stations, tram stops, and other public transport access points within a 2km radius of the property's postcode. Derived from the NaPTAN (National Public Transport Access Nodes) dataset.",
description: "Number of train or tube stations within 1km",
detail: "Count of rail stations and Tube/metro/tram stops within a 1km radius of the property's postcode. Derived from the NaPTAN (National Public Transport Access Nodes) dataset. Does not include bus stops.",
source: "naptan",
prefix: "",
suffix: "",
@ -409,6 +341,23 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
modes: &[],
linked: "",
},
FeatureConfig {
name: "Distance to nearest train or tube station (km)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 0.1,
description: "Distance to the closest train or tube station",
detail: "Straight-line distance in kilometres from the property's postcode centroid to the nearest rail station or Tube/metro/tram stop. Derived from the NaPTAN (National Public Transport Access Nodes) dataset.",
source: "naptan",
prefix: "",
suffix: " km",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
],
},
FeatureGroup {
@ -906,14 +855,31 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
linked: "",
},
FeatureConfig {
name: "% Asian",
name: "% South Asian",
bounds: Bounds::Fixed {
min: 0.0,
max: 100.0,
},
step: 1.0,
description: "Percentage of population identifying as Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Asian or Asian British (Indian, Pakistani, Bangladeshi, Chinese, or any other Asian background).",
description: "Percentage of population identifying as South Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Indian, Pakistani, Bangladeshi, or any other Asian background.",
source: "ethnicity",
prefix: "",
suffix: "%",
raw: false,
absolute: false,
modes: &[],
linked: "",
},
FeatureConfig {
name: "% East Asian",
bounds: Bounds::Fixed {
min: 0.0,
max: 100.0,
},
step: 1.0,
description: "Percentage of population identifying as East Asian",
detail: "From the 2021 Census. Percentage of the local authority population identifying as Chinese.",
source: "ethnicity",
prefix: "",
suffix: "%",
@ -1074,7 +1040,7 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[
EnumFeatureGroup {
name: "Property",
name: "Properties in the area",
features: &[
EnumFeatureConfig {
name: "Listing status",
@ -1084,7 +1050,7 @@ pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[
source: "online-listings",
},
EnumFeatureConfig {
name: "Leashold/Freehold",
name: "Leasehold/Freehold",
order: Some(&["Freehold", "Leasehold"]),
description: "Whether the property is leasehold or freehold",
detail: "From HM Land Registry Price Paid data. Freehold means you own the building and the land it stands on. Leasehold means you own the building but not the land — you have a lease from the freeholder for a set number of years.",

View file

@ -417,16 +417,16 @@ async fn main() -> anyhow::Result<()> {
let state_short_url = state.clone();
let state_ai_filters = state.clone();
let state_streetview = state.clone();
let state_subscription = state.clone();
let state_newsletter = state.clone();
let state_travel_modes = state.clone();
let state_travel_destinations = state.clone();
let state_checkout = state.clone();
let state_stripe_webhook = state.clone();
let state_pricing = state.clone();
let state_invites_create = state.clone();
let state_invite_get = state.clone();
let state_redeem_invite = state.clone();
let state_rightmove = state.clone();
let state_journey = state.clone();
let api = Router::new()
.route(
@ -461,6 +461,14 @@ async fn main() -> anyhow::Result<()> {
"/api/travel-modes",
get(move || routes::get_travel_modes(state_travel_modes.clone())),
)
.route(
"/api/travel-destinations",
get(move |query| routes::get_travel_destinations(state_travel_destinations.clone(), query)),
)
.route(
"/api/journey",
get(move |query| routes::get_journey(state_journey.clone(), query)),
)
.route(
"/api/hexagon-properties",
get(move |ext, query| {
@ -502,16 +510,6 @@ async fn main() -> anyhow::Result<()> {
"/api/streetview",
get(move |query| routes::get_streetview(state_streetview.clone(), query)),
)
.route(
"/api/rightmove-location",
get(move |query| routes::get_rightmove_typeahead(state_rightmove.clone(), query)),
)
.route(
"/api/subscription",
patch(move |ext, body| {
routes::patch_subscription(state_subscription.clone(), ext, body)
}),
)
.route(
"/api/newsletter",
patch(move |ext, body| {

View file

@ -5,6 +5,7 @@ mod features;
mod hexagon_stats;
pub(crate) mod hexagons;
mod invites;
mod journey;
mod me;
mod pb_proxy;
mod places;
@ -20,10 +21,9 @@ mod streetview;
mod stripe_webhook;
mod newsletter;
pub(crate) mod pricing;
mod rightmove_typeahead;
mod subscription;
mod tiles;
pub(crate) mod travel_time;
mod travel_destinations;
mod travel_modes;
pub use ai_filters::{build_ollama_schema, build_system_prompt, post_ai_filters};
@ -44,10 +44,10 @@ pub use screenshot::{fetch_screenshot_bytes, get_screenshot};
pub use shorten::{get_short_url, post_shorten};
pub use streetview::get_streetview;
pub use invites::{get_invite, post_invites, post_redeem_invite};
pub use journey::get_journey;
pub use newsletter::patch_newsletter;
pub use pricing::get_pricing;
pub use stripe_webhook::post_stripe_webhook;
pub use subscription::patch_subscription;
pub use tiles::{get_style, get_tile, init_tile_reader};
pub use rightmove_typeahead::get_rightmove_typeahead;
pub use travel_destinations::get_travel_destinations;
pub use travel_modes::get_travel_modes;

View file

@ -146,7 +146,7 @@ pub fn build_system_prompt(features: &FeaturesResponse) -> String {
parts.push(
"User: \"cheap freehold house under 400k\"\n\
Output: {\"numeric_filters\": [{\"name\": \"Last known price\", \"bound\": \"max\", \"value\": 400000}], \
\"enum_filters\": [{\"name\": \"Leashold/Freehold\", \"values\": [\"Freehold\"]}, \
\"enum_filters\": [{\"name\": \"Leasehold/Freehold\", \"values\": [\"Freehold\"]}, \
{\"name\": \"Property type\", \"values\": [\"Detached\", \"Semi-Detached\", \"Terraced\"]}], \
\"notes\": \"\"}"
.to_string(),
@ -252,13 +252,13 @@ pub async fn post_ai_filters(
/// ```json
/// {
/// "numeric_filters": [{"name": "Last known price", "bound": "max", "value": 300000}],
/// "enum_filters": [{"name": "Leashold/Freehold", "values": ["Freehold"]}]
/// "enum_filters": [{"name": "Leasehold/Freehold", "values": ["Freehold"]}]
/// }
/// ```
///
/// Output format (FeatureFilters):
/// ```json
/// { "Last known price": [0, 300000], "Leashold/Freehold": ["Freehold"] }
/// { "Last known price": [0, 300000], "Leasehold/Freehold": ["Freehold"] }
/// ```
fn validate_and_convert(raw: &Value, features: &FeaturesResponse) -> Value {
let mut result = serde_json::Map::new();

View file

@ -18,7 +18,7 @@ use crate::parsing::{
bounds_intersect, cell_for_row, h3_cell_bounds, needs_parent, parse_field_indices,
parse_filters, require_bounds, row_passes_filters, validate_h3_resolution,
};
use crate::routes::travel_time::TravelTimeAgg;
use crate::routes::travel_time::{parse_travel_entries, TravelTimeAgg};
use crate::state::AppState;
#[derive(Serialize)]
@ -40,62 +40,6 @@ pub struct HexagonParams {
travel: Option<String>,
}
struct TravelEntry {
mode: String,
slug: String,
use_best: bool,
filter_min: Option<f32>,
filter_max: Option<f32>,
}
/// Parse `travel` param into a list of travel entries.
/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max`
fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mut entries = Vec::new();
let mut seen_keys = Vec::new();
for segment in travel_str.split('|') {
let parts: Vec<&str> = segment.split(':').collect();
if parts.len() < 2 {
return Err(format!(
"each travel entry must be 'mode:slug' or 'mode:slug:min:max', got '{}'",
segment
));
}
let mode = parts[0].trim().to_string();
let slug = parts[1].trim().to_string();
let use_best = parts.len() >= 3 && parts[2].trim() == "best";
let filter_offset = if use_best { 1 } else { 0 };
let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset {
let min: f32 = parts[2 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
let max: f32 = parts[3 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
(Some(min), Some(max))
} else {
(None, None)
};
let key = format!("{}:{}", mode, slug);
if seen_keys.contains(&key) {
return Err(format!("duplicate travel entry '{}'", key));
}
seen_keys.push(key);
entries.push(TravelEntry {
mode,
slug,
use_best,
filter_min,
filter_max,
});
}
Ok(entries)
}
/// Build feature maps from aggregated cell data, filtering to only cells that intersect the query bounds.
#[allow(clippy::too_many_arguments)]