lgtm
This commit is contained in:
parent
11711c57e6
commit
81a16f543c
21 changed files with 29072 additions and 1913 deletions
File diff suppressed because one or more lines are too long
6
check.sh
6
check.sh
|
|
@ -12,11 +12,7 @@ step() {
|
||||||
|
|
||||||
step "Python lint: ruff" uv run ruff check .
|
step "Python lint: ruff" uv run ruff check .
|
||||||
step "Python dependency lint: deptry" uv run deptry .
|
step "Python dependency lint: deptry" uv run deptry .
|
||||||
step "Python unit tests" uv run pytest \
|
step "Python unit tests" uv run pytest pipeline
|
||||||
pipeline/utils/test_haversine.py \
|
|
||||||
pipeline/utils/test_poi_counts.py \
|
|
||||||
pipeline/download/test_naptan.py \
|
|
||||||
pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
|
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/frontend"
|
cd "$ROOT_DIR/frontend"
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,19 @@ const HOME_SECTION_HEADING_CLASS =
|
||||||
const HOME_BODY_CLASS = 'text-base leading-relaxed text-warm-600 dark:text-warm-400';
|
const HOME_BODY_CLASS = 'text-base leading-relaxed text-warm-600 dark:text-warm-400';
|
||||||
const HOME_PRIMARY_BUTTON_CLASS =
|
const HOME_PRIMARY_BUTTON_CLASS =
|
||||||
'bg-coral-500 text-white rounded-lg font-semibold hover:bg-coral-600 transition-colors text-base shadow-lg shadow-coral-500/25 text-center';
|
'bg-coral-500 text-white rounded-lg font-semibold hover:bg-coral-600 transition-colors text-base shadow-lg shadow-coral-500/25 text-center';
|
||||||
const PRODUCT_DEMO_VIDEO_SRC = '/video/recording.mp4';
|
const PRODUCT_DEMO_VIDEO_BY_LANGUAGE: Record<string, string> = {
|
||||||
const PRODUCT_DEMO_POSTER_SRC = '/video/recording.jpg';
|
en: 'recording',
|
||||||
|
de: 'recording-de',
|
||||||
|
zh: 'recording-zh',
|
||||||
|
hi: 'recording-hi',
|
||||||
|
};
|
||||||
const PRODUCT_DEMO_SECTION_ID = 'product-demo-video';
|
const PRODUCT_DEMO_SECTION_ID = 'product-demo-video';
|
||||||
|
|
||||||
|
function getProductDemoSlug(language: string | undefined): string {
|
||||||
|
const code = language?.toLowerCase().split('-')[0] ?? 'en';
|
||||||
|
return PRODUCT_DEMO_VIDEO_BY_LANGUAGE[code] ?? PRODUCT_DEMO_VIDEO_BY_LANGUAGE.en;
|
||||||
|
}
|
||||||
|
|
||||||
function highlightBrandText(text: string) {
|
function highlightBrandText(text: string) {
|
||||||
const parts = text.split(BRAND_NAME);
|
const parts = text.split(BRAND_NAME);
|
||||||
if (parts.length === 1) return text;
|
if (parts.length === 1) return text;
|
||||||
|
|
@ -37,11 +46,26 @@ function highlightBrandText(text: string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function ProductDemoVideo() {
|
function ProductDemoVideo() {
|
||||||
const { t } = useTranslation();
|
const { t, i18n } = useTranslation();
|
||||||
const sectionRef = useRef<HTMLDivElement | null>(null);
|
const sectionRef = useRef<HTMLDivElement | null>(null);
|
||||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||||
|
const currentVideoSrcRef = useRef<string | null>(null);
|
||||||
const [shouldLoadVideo, setShouldLoadVideo] = useState(false);
|
const [shouldLoadVideo, setShouldLoadVideo] = useState(false);
|
||||||
const [isVideoPlaying, setIsVideoPlaying] = useState(false);
|
const [isVideoPlaying, setIsVideoPlaying] = useState(false);
|
||||||
|
const productDemoSlug = getProductDemoSlug(i18n.language);
|
||||||
|
const productDemoVideoSrc = `/video/${productDemoSlug}.mp4`;
|
||||||
|
const productDemoPosterSrc = `/video/${productDemoSlug}.jpg`;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (currentVideoSrcRef.current === productDemoVideoSrc) return;
|
||||||
|
currentVideoSrcRef.current = productDemoVideoSrc;
|
||||||
|
setIsVideoPlaying(false);
|
||||||
|
|
||||||
|
const video = videoRef.current;
|
||||||
|
if (!video || !shouldLoadVideo) return;
|
||||||
|
video.pause();
|
||||||
|
video.load();
|
||||||
|
}, [productDemoVideoSrc, shouldLoadVideo]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const section = sectionRef.current;
|
const section = sectionRef.current;
|
||||||
|
|
@ -70,8 +94,8 @@ function ProductDemoVideo() {
|
||||||
setShouldLoadVideo(true);
|
setShouldLoadVideo(true);
|
||||||
if (!video) return;
|
if (!video) return;
|
||||||
|
|
||||||
if (!video.getAttribute('src')) {
|
if (video.getAttribute('src') !== productDemoVideoSrc) {
|
||||||
video.src = PRODUCT_DEMO_VIDEO_SRC;
|
video.src = productDemoVideoSrc;
|
||||||
video.load();
|
video.load();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -86,11 +110,14 @@ function ProductDemoVideo() {
|
||||||
ref={sectionRef}
|
ref={sectionRef}
|
||||||
className={`${HOME_SECTION_CONTAINER_CLASS} pt-8 md:pt-12 pb-2`}
|
className={`${HOME_SECTION_CONTAINER_CLASS} pt-8 md:pt-12 pb-2`}
|
||||||
>
|
>
|
||||||
|
<h2 className={`${HOME_SECTION_HEADING_CLASS} mb-5 text-center`}>
|
||||||
|
{t('home.productDemoLabel')}
|
||||||
|
</h2>
|
||||||
<div className="relative overflow-hidden rounded-lg border border-warm-200 bg-navy-950 shadow-sm dark:border-warm-700">
|
<div className="relative overflow-hidden rounded-lg border border-warm-200 bg-navy-950 shadow-sm dark:border-warm-700">
|
||||||
<video
|
<video
|
||||||
ref={videoRef}
|
ref={videoRef}
|
||||||
src={shouldLoadVideo ? PRODUCT_DEMO_VIDEO_SRC : undefined}
|
src={shouldLoadVideo ? productDemoVideoSrc : undefined}
|
||||||
poster={PRODUCT_DEMO_POSTER_SRC}
|
poster={productDemoPosterSrc}
|
||||||
controls
|
controls
|
||||||
playsInline
|
playsInline
|
||||||
preload={shouldLoadVideo ? 'metadata' : 'none'}
|
preload={shouldLoadVideo ? 'metadata' : 'none'}
|
||||||
|
|
@ -349,6 +376,40 @@ export default function HomePage({
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Street-level detail */}
|
||||||
|
<div className={`${HOME_SECTION_CONTAINER_CLASS} pt-10 md:pt-16 pb-2`}>
|
||||||
|
<div className="grid gap-6 md:grid-cols-[minmax(0,0.9fr)_minmax(0,1.1fr)] md:items-start">
|
||||||
|
<div>
|
||||||
|
<h2 className={`${HOME_SECTION_HEADING_CLASS} mb-4`}>{t('home.streetTitle')}</h2>
|
||||||
|
<p className={`${HOME_BODY_CLASS} max-w-2xl`}>{t('home.streetIntro')}</p>
|
||||||
|
</div>
|
||||||
|
<div className="grid gap-4 sm:grid-cols-2">
|
||||||
|
{[
|
||||||
|
{
|
||||||
|
title: t('home.streetCard1Title'),
|
||||||
|
body: t('home.streetCard1Body'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: t('home.streetCard2Title'),
|
||||||
|
body: t('home.streetCard2Body'),
|
||||||
|
},
|
||||||
|
].map((item) => (
|
||||||
|
<div
|
||||||
|
key={item.title}
|
||||||
|
className="rounded-lg border border-warm-200 bg-white/90 p-5 shadow-sm dark:border-warm-700 dark:bg-warm-800/90"
|
||||||
|
>
|
||||||
|
<h3 className="text-base font-bold text-navy-950 dark:text-warm-100">
|
||||||
|
{item.title}
|
||||||
|
</h3>
|
||||||
|
<p className="mt-3 text-sm leading-relaxed text-warm-600 dark:text-warm-400">
|
||||||
|
{item.body}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* Comparison table */}
|
{/* Comparison table */}
|
||||||
<div
|
<div
|
||||||
id="how-it-works"
|
id="how-it-works"
|
||||||
|
|
@ -424,16 +485,25 @@ export default function HomePage({
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</td>
|
</td>
|
||||||
{[row.postcode, row.guides].map((has, j) => (
|
{[row.postcode, row.guides].map((has, j) => {
|
||||||
|
const statusLabel = has ? 'Yes' : 'No';
|
||||||
|
return (
|
||||||
<td
|
<td
|
||||||
key={j}
|
key={j}
|
||||||
|
aria-label={statusLabel}
|
||||||
className={`px-1.5 md:px-3 py-2.5 md:py-3.5 text-center text-base ${has ? 'text-green-500' : 'text-red-500'}`}
|
className={`px-1.5 md:px-3 py-2.5 md:py-3.5 text-center text-base ${has ? 'text-green-500' : 'text-red-500'}`}
|
||||||
>
|
>
|
||||||
{has ? '\u2713' : '\u2717'}
|
<span aria-hidden="true">{has ? '\u2713' : '\u2717'}</span>
|
||||||
|
<span className="sr-only">{statusLabel}</span>
|
||||||
</td>
|
</td>
|
||||||
))}
|
);
|
||||||
<td className="px-1.5 md:px-3 py-2.5 md:py-3.5 text-center text-base text-green-500 bg-teal-50 dark:bg-teal-900/30">
|
})}
|
||||||
✓
|
<td
|
||||||
|
aria-label="Yes"
|
||||||
|
className="px-1.5 md:px-3 py-2.5 md:py-3.5 text-center text-base text-green-500 bg-teal-50 dark:bg-teal-900/30"
|
||||||
|
>
|
||||||
|
<span aria-hidden="true">✓</span>
|
||||||
|
<span className="sr-only">Yes</span>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
))}
|
||||||
|
|
|
||||||
|
|
@ -626,6 +626,9 @@ function EnglandHexMapScreen({ isActive }: { isActive: boolean }) {
|
||||||
value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(),
|
value: SHOWCASE_MAP_TOTAL_COUNT.toLocaleString(),
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
|
<div className="mt-2 text-[10px] font-bold uppercase tracking-wide text-warm-400">
|
||||||
|
{t('home.showcaseStep2Sources')}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -91,11 +91,13 @@ export default function FeatureBrowser({
|
||||||
search.toLowerCase()
|
search.toLowerCase()
|
||||||
));
|
));
|
||||||
|
|
||||||
// Ensure "Transport" group exists first when travel modes should be shown.
|
// Keep "Transport" first because journey and transport proximity controls belong together.
|
||||||
const mergedGrouped = useMemo(() => {
|
const mergedGrouped = useMemo(() => {
|
||||||
if (!showTravelModes) return grouped;
|
const transportGroup = grouped.find((g) => g.name === 'Transport');
|
||||||
if (grouped.some((g) => g.name === 'Transport')) return grouped;
|
const otherGroups = grouped.filter((g) => g.name !== 'Transport');
|
||||||
return [{ name: 'Transport', features: [] }, ...grouped];
|
if (transportGroup) return [transportGroup, ...otherGroups];
|
||||||
|
if (showTravelModes) return [{ name: 'Transport', features: [] }, ...otherGroups];
|
||||||
|
return otherGroups;
|
||||||
}, [grouped, showTravelModes]);
|
}, [grouped, showTravelModes]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,7 @@ export function SliderLabels({
|
||||||
isAtMax,
|
isAtMax,
|
||||||
raw,
|
raw,
|
||||||
feature,
|
feature,
|
||||||
|
showUnit,
|
||||||
onValueChange,
|
onValueChange,
|
||||||
}: {
|
}: {
|
||||||
min: number;
|
min: number;
|
||||||
|
|
@ -91,6 +92,7 @@ export function SliderLabels({
|
||||||
isAtMax?: boolean;
|
isAtMax?: boolean;
|
||||||
raw?: boolean;
|
raw?: boolean;
|
||||||
feature?: FeatureMeta;
|
feature?: FeatureMeta;
|
||||||
|
showUnit?: boolean;
|
||||||
onValueChange?: (v: [number, number]) => void;
|
onValueChange?: (v: [number, number]) => void;
|
||||||
}) {
|
}) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
|
@ -98,7 +100,10 @@ export function SliderLabels({
|
||||||
const leftPct = Math.max(0, Math.min(100, ((value[0] - min) / range) * 100));
|
const leftPct = Math.max(0, Math.min(100, ((value[0] - min) / range) * 100));
|
||||||
const rightPct = Math.max(0, Math.min(100, ((value[1] - min) / range) * 100));
|
const rightPct = Math.max(0, Math.min(100, ((value[1] - min) / range) * 100));
|
||||||
const labels = displayValues || value;
|
const labels = displayValues || value;
|
||||||
const labelFormat = feature?.suffix === '%' ? { raw, suffix: feature.suffix } : raw;
|
const shouldShowUnit = Boolean(feature && (showUnit || feature.suffix === '%'));
|
||||||
|
const labelFormat = shouldShowUnit
|
||||||
|
? { raw, prefix: feature?.prefix, suffix: feature?.suffix }
|
||||||
|
: raw;
|
||||||
|
|
||||||
const minLabel = isAtMin ? t('common.min') : formatFilterValue(labels[0], labelFormat);
|
const minLabel = isAtMin ? t('common.min') : formatFilterValue(labels[0], labelFormat);
|
||||||
const maxLabel = isAtMax ? t('common.max') : formatFilterValue(labels[1], labelFormat);
|
const maxLabel = isAtMax ? t('common.max') : formatFilterValue(labels[1], labelFormat);
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,8 @@
|
||||||
<meta name="theme-color" content="#fafaf9" media="(prefers-color-scheme: light)" />
|
<meta name="theme-color" content="#fafaf9" media="(prefers-color-scheme: light)" />
|
||||||
<meta name="theme-color" content="#0a0e1a" media="(prefers-color-scheme: dark)" />
|
<meta name="theme-color" content="#0a0e1a" media="(prefers-color-scheme: dark)" />
|
||||||
<meta name="referrer" content="no-referrer" />
|
<meta name="referrer" content="no-referrer" />
|
||||||
<title>Perfect Postcode - Find where to buy before browsing listings</title>
|
<title>Find the best postcodes and areas to live in England | Perfect Postcode</title>
|
||||||
<meta name="description" content="Search every postcode by budget, commute, schools, safety, noise, broadband, prices and more. Build a better home-buying shortlist before viewings." />
|
<meta name="description" content="Discover where to live by comparing England postcodes by budget, commute, schools, crime, noise, broadband, property prices and local amenities before viewing homes." />
|
||||||
<meta name="x-og-placeholder" content="__PERFECT_POSTCODE_OG_TAGS__" />
|
<meta name="x-og-placeholder" content="__PERFECT_POSTCODE_OG_TAGS__" />
|
||||||
<script>
|
<script>
|
||||||
(function() {
|
(function() {
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import { createElectionVoteShareFilterKey } from './election-filter';
|
||||||
import { createEthnicityFilterKey } from './ethnicity-filter';
|
import { createEthnicityFilterKey } from './ethnicity-filter';
|
||||||
import {
|
import {
|
||||||
POI_COUNT_2KM_FILTER_NAME,
|
POI_COUNT_2KM_FILTER_NAME,
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
createPoiDistanceFilterKey,
|
createPoiDistanceFilterKey,
|
||||||
createPoiFilterKey,
|
createPoiFilterKey,
|
||||||
} from './poi-distance-filter';
|
} from './poi-distance-filter';
|
||||||
|
|
@ -225,13 +226,13 @@ describe('url-state', () => {
|
||||||
|
|
||||||
it('round-trips repeated amenity distance filters with dedicated URL params', () => {
|
it('round-trips repeated amenity distance filters with dedicated URL params', () => {
|
||||||
const park = createPoiDistanceFilterKey('Distance to nearest park (km)', 3);
|
const park = createPoiDistanceFilterKey('Distance to nearest park (km)', 3);
|
||||||
const tesco = createPoiDistanceFilterKey('Distance to nearest Tesco (km)', 4);
|
const grocery = createPoiDistanceFilterKey('Distance to nearest grocery store (km)', 4);
|
||||||
|
|
||||||
const params = stateToParams(
|
const params = stateToParams(
|
||||||
null,
|
null,
|
||||||
{
|
{
|
||||||
[park]: [0, 0.4],
|
[park]: [0, 0.4],
|
||||||
[tesco]: [0, 1.5],
|
[grocery]: [0, 1.5],
|
||||||
},
|
},
|
||||||
[],
|
[],
|
||||||
new Set(),
|
new Set(),
|
||||||
|
|
@ -240,7 +241,7 @@ describe('url-state', () => {
|
||||||
|
|
||||||
expect(params.getAll('amenityDistance')).toEqual([
|
expect(params.getAll('amenityDistance')).toEqual([
|
||||||
'Distance%20to%20nearest%20park%20(km):0:0.4',
|
'Distance%20to%20nearest%20park%20(km):0:0.4',
|
||||||
'Distance%20to%20nearest%20Tesco%20(km):0:1.5',
|
'Distance%20to%20nearest%20grocery%20store%20(km):0:1.5',
|
||||||
]);
|
]);
|
||||||
expect(params.getAll('filter')).toEqual([]);
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
|
@ -249,7 +250,60 @@ describe('url-state', () => {
|
||||||
|
|
||||||
expect(state.filters).toEqual({
|
expect(state.filters).toEqual({
|
||||||
[createPoiDistanceFilterKey('Distance to nearest park (km)', 0)]: [0, 0.4],
|
[createPoiDistanceFilterKey('Distance to nearest park (km)', 0)]: [0, 0.4],
|
||||||
[createPoiDistanceFilterKey('Distance to nearest Tesco (km)', 1)]: [0, 1.5],
|
[createPoiDistanceFilterKey('Distance to nearest grocery store (km)', 1)]: [0, 1.5],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('round-trips transport distance filters with dedicated URL params', () => {
|
||||||
|
const busStop = createPoiFilterKey(
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
'Distance to nearest amenity (Bus stop) (km)',
|
||||||
|
3
|
||||||
|
);
|
||||||
|
|
||||||
|
const params = stateToParams(
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
[busStop]: [0, 0.3],
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
new Set(),
|
||||||
|
'area'
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(params.getAll('transportDistance')).toEqual([
|
||||||
|
'Distance%20to%20nearest%20amenity%20(Bus%20stop)%20(km):0:0.3',
|
||||||
|
]);
|
||||||
|
expect(params.getAll('amenityDistance')).toEqual([]);
|
||||||
|
expect(params.getAll('filter')).toEqual([]);
|
||||||
|
|
||||||
|
window.history.replaceState({}, '', `/?${params.toString()}`);
|
||||||
|
const state = parseUrlState();
|
||||||
|
|
||||||
|
expect(state.filters).toEqual({
|
||||||
|
[createPoiFilterKey(
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
'Distance to nearest amenity (Bus stop) (km)',
|
||||||
|
0
|
||||||
|
)]: [0, 0.3],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('migrates legacy transport distance amenity params into transport filters', () => {
|
||||||
|
window.history.replaceState(
|
||||||
|
{},
|
||||||
|
'',
|
||||||
|
'/?amenityDistance=Distance%20to%20nearest%20amenity%20(Bus%20stop)%20(km):0:0.3'
|
||||||
|
);
|
||||||
|
|
||||||
|
const state = parseUrlState();
|
||||||
|
|
||||||
|
expect(state.filters).toEqual({
|
||||||
|
[createPoiFilterKey(
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
'Distance to nearest amenity (Bus stop) (km)',
|
||||||
|
0
|
||||||
|
)]: [0, 0.3],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -279,8 +333,9 @@ describe('url-state', () => {
|
||||||
const state = parseUrlState();
|
const state = parseUrlState();
|
||||||
|
|
||||||
expect(state.filters).toEqual({
|
expect(state.filters).toEqual({
|
||||||
[createPoiFilterKey(POI_COUNT_2KM_FILTER_NAME, 'Number of amenities (Cafe) within 2km', 0)]:
|
[createPoiFilterKey(POI_COUNT_2KM_FILTER_NAME, 'Number of amenities (Cafe) within 2km', 0)]: [
|
||||||
[2, 8],
|
2, 8,
|
||||||
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,13 +38,12 @@ import {
|
||||||
} from './ethnicity-filter';
|
} from './ethnicity-filter';
|
||||||
import {
|
import {
|
||||||
POI_DISTANCE_FILTER_NAME,
|
POI_DISTANCE_FILTER_NAME,
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
POI_COUNT_2KM_FILTER_NAME,
|
POI_COUNT_2KM_FILTER_NAME,
|
||||||
POI_COUNT_5KM_FILTER_NAME,
|
POI_COUNT_5KM_FILTER_NAME,
|
||||||
createPoiFilterKey,
|
createPoiFilterKey,
|
||||||
createPoiDistanceFilterKey,
|
|
||||||
getPoiDistanceFeatureName,
|
getPoiDistanceFeatureName,
|
||||||
getPoiFilterName,
|
getPoiFilterName,
|
||||||
isPoiDistanceFeatureName,
|
|
||||||
isPoiDistanceFilterName,
|
isPoiDistanceFilterName,
|
||||||
type PoiFilterName,
|
type PoiFilterName,
|
||||||
} from './poi-distance-filter';
|
} from './poi-distance-filter';
|
||||||
|
|
@ -68,6 +67,7 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
const voteShareParams = params.getAll('voteShare');
|
const voteShareParams = params.getAll('voteShare');
|
||||||
const ethnicityParams = params.getAll('ethnicity');
|
const ethnicityParams = params.getAll('ethnicity');
|
||||||
const amenityDistanceParams = params.getAll('amenityDistance');
|
const amenityDistanceParams = params.getAll('amenityDistance');
|
||||||
|
const transportDistanceParams = params.getAll('transportDistance');
|
||||||
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
||||||
const amenityCount5KmParams = params.getAll('amenityCount5km');
|
const amenityCount5KmParams = params.getAll('amenityCount5km');
|
||||||
if (
|
if (
|
||||||
|
|
@ -77,6 +77,7 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
voteShareParams.length === 0 &&
|
voteShareParams.length === 0 &&
|
||||||
ethnicityParams.length === 0 &&
|
ethnicityParams.length === 0 &&
|
||||||
amenityDistanceParams.length === 0 &&
|
amenityDistanceParams.length === 0 &&
|
||||||
|
transportDistanceParams.length === 0 &&
|
||||||
amenityCount2KmParams.length === 0 &&
|
amenityCount2KmParams.length === 0 &&
|
||||||
amenityCount5KmParams.length === 0
|
amenityCount5KmParams.length === 0
|
||||||
) {
|
) {
|
||||||
|
|
@ -159,44 +160,51 @@ function parseFilters(params: URLSearchParams): FeatureFilters {
|
||||||
filters[createEthnicityFilterKey(featureName, index)] = [min, max];
|
filters[createEthnicityFilterKey(featureName, index)] = [min, max];
|
||||||
});
|
});
|
||||||
|
|
||||||
amenityDistanceParams.forEach((entry, index) => {
|
const parsePoiParams = (entries: string[], filterName: PoiFilterName, startIndex: number) => {
|
||||||
const parts = entry.split(':');
|
|
||||||
if (parts.length < 3) return;
|
|
||||||
const featureName = decodeURIComponent(parts.slice(0, -2).join(':'));
|
|
||||||
const min = Number(parts[parts.length - 2]);
|
|
||||||
const max = Number(parts[parts.length - 1]);
|
|
||||||
if (!isPoiDistanceFeatureName(featureName) || isNaN(min) || isNaN(max)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
filters[createPoiDistanceFilterKey(featureName, index)] = [min, max];
|
|
||||||
});
|
|
||||||
|
|
||||||
const parsePoiCountParams = (
|
|
||||||
entries: string[],
|
|
||||||
filterName: PoiFilterName,
|
|
||||||
startIndex: number
|
|
||||||
) => {
|
|
||||||
entries.forEach((entry, index) => {
|
entries.forEach((entry, index) => {
|
||||||
const parts = entry.split(':');
|
const parts = entry.split(':');
|
||||||
if (parts.length < 3) return;
|
if (parts.length < 3) return;
|
||||||
const featureName = decodeURIComponent(parts.slice(0, -2).join(':'));
|
const featureName = decodeURIComponent(parts.slice(0, -2).join(':'));
|
||||||
const min = Number(parts[parts.length - 2]);
|
const min = Number(parts[parts.length - 2]);
|
||||||
const max = Number(parts[parts.length - 1]);
|
const max = Number(parts[parts.length - 1]);
|
||||||
if (getPoiFilterName(featureName) !== filterName || isNaN(min) || isNaN(max)) {
|
const targetFilterName = getPoiFilterName(featureName);
|
||||||
|
const canMigrateTransportDistance =
|
||||||
|
filterName === POI_DISTANCE_FILTER_NAME &&
|
||||||
|
targetFilterName === TRANSPORT_DISTANCE_FILTER_NAME;
|
||||||
|
if (
|
||||||
|
!targetFilterName ||
|
||||||
|
(targetFilterName !== filterName && !canMigrateTransportDistance) ||
|
||||||
|
isNaN(min) ||
|
||||||
|
isNaN(max)
|
||||||
|
) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
filters[createPoiFilterKey(filterName, featureName, startIndex + index)] = [min, max];
|
filters[createPoiFilterKey(targetFilterName, featureName, startIndex + index)] = [min, max];
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const parsePoiCountParams = (
|
||||||
|
entries: string[],
|
||||||
|
filterName: PoiFilterName,
|
||||||
|
startIndex: number
|
||||||
|
) => {
|
||||||
|
parsePoiParams(entries, filterName, startIndex);
|
||||||
|
};
|
||||||
|
parsePoiParams(amenityDistanceParams, POI_DISTANCE_FILTER_NAME, 0);
|
||||||
|
parsePoiParams(
|
||||||
|
transportDistanceParams,
|
||||||
|
TRANSPORT_DISTANCE_FILTER_NAME,
|
||||||
|
amenityDistanceParams.length
|
||||||
|
);
|
||||||
parsePoiCountParams(
|
parsePoiCountParams(
|
||||||
amenityCount2KmParams,
|
amenityCount2KmParams,
|
||||||
POI_COUNT_2KM_FILTER_NAME,
|
POI_COUNT_2KM_FILTER_NAME,
|
||||||
amenityDistanceParams.length
|
amenityDistanceParams.length + transportDistanceParams.length
|
||||||
);
|
);
|
||||||
parsePoiCountParams(
|
parsePoiCountParams(
|
||||||
amenityCount5KmParams,
|
amenityCount5KmParams,
|
||||||
POI_COUNT_5KM_FILTER_NAME,
|
POI_COUNT_5KM_FILTER_NAME,
|
||||||
amenityDistanceParams.length + amenityCount2KmParams.length
|
amenityDistanceParams.length + transportDistanceParams.length + amenityCount2KmParams.length
|
||||||
);
|
);
|
||||||
|
|
||||||
return filters;
|
return filters;
|
||||||
|
|
@ -349,11 +357,10 @@ export function stateToParams(
|
||||||
? 'amenityCount2km'
|
? 'amenityCount2km'
|
||||||
: filterName === POI_COUNT_5KM_FILTER_NAME
|
: filterName === POI_COUNT_5KM_FILTER_NAME
|
||||||
? 'amenityCount5km'
|
? 'amenityCount5km'
|
||||||
|
: filterName === TRANSPORT_DISTANCE_FILTER_NAME
|
||||||
|
? 'transportDistance'
|
||||||
: 'amenityDistance';
|
: 'amenityDistance';
|
||||||
params.append(
|
params.append(paramName, `${encodeURIComponent(amenityDistanceFeatureName)}:${min}:${max}`);
|
||||||
paramName,
|
|
||||||
`${encodeURIComponent(amenityDistanceFeatureName)}:${min}:${max}`
|
|
||||||
);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -410,6 +417,7 @@ export function summarizeParams(queryString: string): string {
|
||||||
const voteShareParams = params.getAll('voteShare');
|
const voteShareParams = params.getAll('voteShare');
|
||||||
const ethnicityParams = params.getAll('ethnicity');
|
const ethnicityParams = params.getAll('ethnicity');
|
||||||
const amenityDistanceParams = params.getAll('amenityDistance');
|
const amenityDistanceParams = params.getAll('amenityDistance');
|
||||||
|
const transportDistanceParams = params.getAll('transportDistance');
|
||||||
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
const amenityCount2KmParams = params.getAll('amenityCount2km');
|
||||||
const amenityCount5KmParams = params.getAll('amenityCount5km');
|
const amenityCount5KmParams = params.getAll('amenityCount5km');
|
||||||
if (
|
if (
|
||||||
|
|
@ -419,6 +427,7 @@ export function summarizeParams(queryString: string): string {
|
||||||
voteShareParams.length > 0 ||
|
voteShareParams.length > 0 ||
|
||||||
ethnicityParams.length > 0 ||
|
ethnicityParams.length > 0 ||
|
||||||
amenityDistanceParams.length > 0 ||
|
amenityDistanceParams.length > 0 ||
|
||||||
|
transportDistanceParams.length > 0 ||
|
||||||
amenityCount2KmParams.length > 0 ||
|
amenityCount2KmParams.length > 0 ||
|
||||||
amenityCount5KmParams.length > 0
|
amenityCount5KmParams.length > 0
|
||||||
) {
|
) {
|
||||||
|
|
@ -429,7 +438,8 @@ export function summarizeParams(queryString: string): string {
|
||||||
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
|
if (isSpecificCrimeFeatureName(name)) return SPECIFIC_CRIMES_FILTER_NAME;
|
||||||
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
|
if (isElectionVoteShareFeatureName(name)) return ELECTION_VOTE_SHARE_FILTER_NAME;
|
||||||
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
|
if (isEthnicityFeatureName(name)) return ETHNICITIES_FILTER_NAME;
|
||||||
if (isPoiDistanceFeatureName(name)) return POI_DISTANCE_FILTER_NAME;
|
const poiFilterName = getPoiFilterName(name);
|
||||||
|
if (poiFilterName) return poiFilterName;
|
||||||
return name;
|
return name;
|
||||||
})
|
})
|
||||||
.filter((n) => n);
|
.filter((n) => n);
|
||||||
|
|
@ -446,6 +456,9 @@ export function summarizeParams(queryString: string): string {
|
||||||
for (let i = 0; i < amenityDistanceParams.length; i++) {
|
for (let i = 0; i < amenityDistanceParams.length; i++) {
|
||||||
filterNames.push(POI_DISTANCE_FILTER_NAME);
|
filterNames.push(POI_DISTANCE_FILTER_NAME);
|
||||||
}
|
}
|
||||||
|
for (let i = 0; i < transportDistanceParams.length; i++) {
|
||||||
|
filterNames.push(TRANSPORT_DISTANCE_FILTER_NAME);
|
||||||
|
}
|
||||||
for (let i = 0; i < amenityCount2KmParams.length; i++) {
|
for (let i = 0; i < amenityCount2KmParams.length; i++) {
|
||||||
filterNames.push(POI_COUNT_2KM_FILTER_NAME);
|
filterNames.push(POI_COUNT_2KM_FILTER_NAME);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,14 +27,14 @@ fn type_rank(place_type: &str) -> u8 {
|
||||||
"town" => 1,
|
"town" => 1,
|
||||||
"village" => 2,
|
"village" => 2,
|
||||||
"suburb" | "neighbourhood" | "quarter" | "borough" | "locality" => 3,
|
"suburb" | "neighbourhood" | "quarter" | "borough" | "locality" => 3,
|
||||||
"station" => 4,
|
"station" | "university" => 4,
|
||||||
"hamlet" | "isolated_dwelling" | "island" => 5,
|
"hamlet" | "isolated_dwelling" | "island" => 5,
|
||||||
_ => 6,
|
_ => 6,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_travel_destination_type(place_type: &str) -> bool {
|
pub fn is_travel_destination_type(place_type: &str) -> bool {
|
||||||
matches!(place_type, "city" | "station")
|
matches!(place_type, "city" | "station" | "university")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn normalize_search_text(text: &str) -> String {
|
pub fn normalize_search_text(text: &str) -> String {
|
||||||
|
|
|
||||||
|
|
@ -161,17 +161,17 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
||||||
absolute: false,
|
absolute: false,
|
||||||
}),
|
}),
|
||||||
Feature::Numeric(FeatureConfig {
|
Feature::Numeric(FeatureConfig {
|
||||||
name: "Street tree density (%)",
|
name: "Street tree density percentile",
|
||||||
bounds: Bounds::Fixed {
|
bounds: Bounds::Fixed {
|
||||||
min: 0.0,
|
min: 0.0,
|
||||||
max: 100.0,
|
max: 100.0,
|
||||||
},
|
},
|
||||||
step: 1.0,
|
step: 1.0,
|
||||||
description: "Estimated tree canopy density on the property's street",
|
description: "Estimated tree canopy coverage percentile for the property's street",
|
||||||
detail: "Approximate street-level tree density derived from Forest Research's 2025 Trees Outside Woodland map. Tree canopy polygons for lone trees and groups of trees are counted within 50m of postcode centroids, then averaged across Price Paid addresses on the same street. This is a street proxy, not an exact address-to-road-segment measurement.",
|
detail: "Approximate street-level tree coverage derived from Forest Research's 2025 Trees Outside Woodland map. Tree canopy polygons for lone trees and groups of trees are counted within 50m of postcode centroids, averaged across Price Paid addresses on the same street, then converted to a percentile across English streets. This is a street proxy, not an exact address-to-road-segment measurement.",
|
||||||
source: "forest-research-tow",
|
source: "forest-research-tow",
|
||||||
prefix: "",
|
prefix: "",
|
||||||
suffix: "%",
|
suffix: "",
|
||||||
raw: false,
|
raw: false,
|
||||||
absolute: true,
|
absolute: true,
|
||||||
}),
|
}),
|
||||||
|
|
@ -1013,66 +1013,6 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
||||||
raw: false,
|
raw: false,
|
||||||
absolute: false,
|
absolute: false,
|
||||||
}),
|
}),
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Distance to nearest tube station (km)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 0.1,
|
|
||||||
description: "Distance to the closest Tube, metro, tram, or DLR stop",
|
|
||||||
detail: "Straight-line distance in kilometres from the postcode to the nearest NaPTAN station classified as Tube, metro, tram, or DLR.",
|
|
||||||
source: "naptan",
|
|
||||||
prefix: "",
|
|
||||||
suffix: " km",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Distance to nearest rail station (km)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 0.1,
|
|
||||||
description: "Distance to the closest National Rail station",
|
|
||||||
detail: "Straight-line distance in kilometres from the postcode to the nearest NaPTAN railway station.",
|
|
||||||
source: "naptan",
|
|
||||||
prefix: "",
|
|
||||||
suffix: " km",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Distance to nearest Waitrose (km)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 0.1,
|
|
||||||
description: "Distance to the closest Waitrose store",
|
|
||||||
detail: "Straight-line distance in kilometres from the postcode to the nearest Waitrose or Little Waitrose store in the GEOLYTIX Grocery Retail Points dataset.",
|
|
||||||
source: "geolytix-retail-points",
|
|
||||||
prefix: "",
|
|
||||||
suffix: " km",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
|
||||||
name: "Distance to nearest Tesco (km)",
|
|
||||||
bounds: Bounds::Percentile {
|
|
||||||
low: 2.0,
|
|
||||||
high: 98.0,
|
|
||||||
},
|
|
||||||
step: 0.1,
|
|
||||||
description: "Distance to the closest Tesco store",
|
|
||||||
detail: "Straight-line distance in kilometres from the postcode to the nearest Tesco store in the GEOLYTIX Grocery Retail Points dataset.",
|
|
||||||
source: "geolytix-retail-points",
|
|
||||||
prefix: "",
|
|
||||||
suffix: " km",
|
|
||||||
raw: false,
|
|
||||||
absolute: false,
|
|
||||||
}),
|
|
||||||
Feature::Numeric(FeatureConfig {
|
Feature::Numeric(FeatureConfig {
|
||||||
name: "Distance to nearest cafe (km)",
|
name: "Distance to nearest cafe (km)",
|
||||||
bounds: Bounds::Percentile {
|
bounds: Bounds::Percentile {
|
||||||
|
|
|
||||||
|
|
@ -559,6 +559,10 @@ async fn main() -> anyhow::Result<()> {
|
||||||
post(routes::post_ai_filters).layer(ConcurrencyLimitLayer::new(5)),
|
post(routes::post_ai_filters).layer(ConcurrencyLimitLayer::new(5)),
|
||||||
)
|
)
|
||||||
.route("/api/streetview", get(routes::get_streetview))
|
.route("/api/streetview", get(routes::get_streetview))
|
||||||
|
.route(
|
||||||
|
"/api/rightmove-search",
|
||||||
|
get(routes::get_rightmove_redirect).layer(ConcurrencyLimitLayer::new(10)),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/newsletter",
|
"/api/newsletter",
|
||||||
patch(routes::patch_newsletter).layer(ConcurrencyLimitLayer::new(10)),
|
patch(routes::patch_newsletter).layer(ConcurrencyLimitLayer::new(10)),
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ pub use fields::{
|
||||||
parse_enum_dist, parse_field_indices, parse_field_indices_with_poi, parse_field_set,
|
parse_enum_dist, parse_field_indices, parse_field_indices_with_poi, parse_field_set,
|
||||||
};
|
};
|
||||||
pub use filters::{
|
pub use filters::{
|
||||||
count_filter_impacts, parse_filters, parse_filters_with_poi, row_passes_filters,
|
count_filter_impacts, count_filter_rejections, parse_filters, parse_filters_with_poi,
|
||||||
row_passes_poi_filters, ParsedEnumFilter, ParsedFilter, ParsedPoiFilter,
|
row_passes_filters, row_passes_poi_filters, ParsedEnumFilter, ParsedFilter, ParsedPoiFilter,
|
||||||
};
|
};
|
||||||
pub use h3::{cell_for_row, cell_for_row_cached, needs_parent, validate_h3_resolution};
|
pub use h3::{cell_for_row, cell_for_row_cached, needs_parent, validate_h3_resolution};
|
||||||
|
|
|
||||||
|
|
@ -251,16 +251,13 @@ pub fn row_passes_poi_filters(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Single-pass marginal impact counting.
|
/// Single-pass per-filter rejection counting.
|
||||||
///
|
///
|
||||||
/// Returns `(total_passing, impacts)` where `impacts[i]` is how many MORE rows
|
/// Returns `(total_passing, rejections)` where `rejections[i]` is how many rows
|
||||||
/// would pass if the i-th filter (numeric first, then enum) were removed.
|
/// the i-th filter (numeric first, then enum) rejects.
|
||||||
///
|
///
|
||||||
/// For each row we record which filters reject it:
|
/// Rows rejected by multiple filters are counted for every rejecting filter.
|
||||||
/// - 0 failures → passes (counted in `total_passing`)
|
pub fn count_filter_rejections(
|
||||||
/// - exactly 1 failure → that filter's marginal cost (counted in `impacts[i]`)
|
|
||||||
/// - 2+ failures → removing any single filter won't recover it (ignored)
|
|
||||||
pub fn count_filter_impacts(
|
|
||||||
filters: &[ParsedFilter],
|
filters: &[ParsedFilter],
|
||||||
enum_filters: &[ParsedEnumFilter],
|
enum_filters: &[ParsedEnumFilter],
|
||||||
feature_data: &[u16],
|
feature_data: &[u16],
|
||||||
|
|
@ -269,45 +266,45 @@ pub fn count_filter_impacts(
|
||||||
) -> (u32, Vec<u32>) {
|
) -> (u32, Vec<u32>) {
|
||||||
let n = filters.len() + enum_filters.len();
|
let n = filters.len() + enum_filters.len();
|
||||||
let mut total_passing: u32 = 0;
|
let mut total_passing: u32 = 0;
|
||||||
let mut impacts = vec![0u32; n];
|
let mut rejections = vec![0u32; n];
|
||||||
|
|
||||||
for row_idx in rows {
|
for row_idx in rows {
|
||||||
let base = row_idx as usize * num_features;
|
let base = row_idx as usize * num_features;
|
||||||
let mut fail_count: u32 = 0;
|
let mut passes_all = true;
|
||||||
let mut fail_index: usize = 0;
|
|
||||||
|
|
||||||
for (i, f) in filters.iter().enumerate() {
|
for (i, f) in filters.iter().enumerate() {
|
||||||
let raw = feature_data[base + f.feat_idx];
|
let raw = feature_data[base + f.feat_idx];
|
||||||
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
||||||
fail_count += 1;
|
rejections[i] += 1;
|
||||||
fail_index = i;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if fail_count <= 1 {
|
|
||||||
for (i, f) in enum_filters.iter().enumerate() {
|
for (i, f) in enum_filters.iter().enumerate() {
|
||||||
let raw = feature_data[base + f.feat_idx];
|
let raw = feature_data[base + f.feat_idx];
|
||||||
if raw == NAN_U16 || !f.allowed.contains(&raw) {
|
if raw == NAN_U16 || !f.allowed.contains(&raw) {
|
||||||
fail_count += 1;
|
rejections[filters.len() + i] += 1;
|
||||||
fail_index = filters.len() + i;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match fail_count {
|
if passes_all {
|
||||||
0 => total_passing += 1,
|
total_passing += 1;
|
||||||
1 => impacts[fail_index] += 1,
|
|
||||||
_ => {}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
(total_passing, impacts)
|
(total_passing, rejections)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Backward-compatible name retained for existing callers.
|
||||||
|
pub fn count_filter_impacts(
|
||||||
|
filters: &[ParsedFilter],
|
||||||
|
enum_filters: &[ParsedEnumFilter],
|
||||||
|
feature_data: &[u16],
|
||||||
|
num_features: usize,
|
||||||
|
rows: impl Iterator<Item = u32>,
|
||||||
|
) -> (u32, Vec<u32>) {
|
||||||
|
count_filter_rejections(filters, enum_filters, feature_data, num_features, rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -456,14 +453,14 @@ mod tests {
|
||||||
let tq = test_quant(3, 2);
|
let tq = test_quant(3, 2);
|
||||||
let poi_tq = test_quant(2, 2);
|
let poi_tq = test_quant(2, 2);
|
||||||
let poi_map: FxHashMap<String, usize> = [
|
let poi_map: FxHashMap<String, usize> = [
|
||||||
("Distance to nearest cafe POI (km)".into(), 0),
|
("Distance to nearest amenity (Cafe) (km)".into(), 0),
|
||||||
("Number of cafe POIs within 2km".into(), 1),
|
("Number of amenities (Cafe) within 2km".into(), 1),
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let (numeric, enums, poi) = parse_filters_with_poi(
|
let (numeric, enums, poi) = parse_filters_with_poi(
|
||||||
Some("price:100:500;;rating:A;;Distance to nearest cafe POI (km):0:1.5"),
|
Some("price:100:500;;rating:A;;Distance to nearest amenity (Cafe) (km):0:1.5"),
|
||||||
&feature_name_to_index(),
|
&feature_name_to_index(),
|
||||||
&enum_values(),
|
&enum_values(),
|
||||||
&tq.as_ref(),
|
&tq.as_ref(),
|
||||||
|
|
@ -817,7 +814,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn filter_impacts_single_pass() {
|
fn filter_rejections_include_rows_that_fail_multiple_filters() {
|
||||||
// 2 numeric features, 4 rows:
|
// 2 numeric features, 4 rows:
|
||||||
// row 0: price=150, area=100 → passes both
|
// row 0: price=150, area=100 → passes both
|
||||||
// row 1: price=600, area=100 → fails price only
|
// row 1: price=600, area=100 → fails price only
|
||||||
|
|
@ -847,20 +844,20 @@ mod tests {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
let (total, impacts) = count_filter_impacts(&filters, &[], &feature_data, 2, 0..4u32);
|
let (total, rejections) = count_filter_rejections(&filters, &[], &feature_data, 2, 0..4u32);
|
||||||
|
|
||||||
assert_eq!(total, 1); // only row 0 passes
|
assert_eq!(total, 1); // only row 0 passes
|
||||||
assert_eq!(impacts[0], 1); // row 1 fails price only
|
assert_eq!(rejections[0], 2); // rows 1 and 3 fail price
|
||||||
assert_eq!(impacts[1], 1); // row 2 fails area only
|
assert_eq!(rejections[1], 2); // rows 2 and 3 fail area
|
||||||
// row 3 fails both → not counted
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn filter_impacts_with_enum() {
|
fn filter_rejections_with_enum() {
|
||||||
// 1 numeric + 1 enum, 3 rows:
|
// 1 numeric + 1 enum, 4 rows:
|
||||||
// row 0: price=150, type=0(A) → passes both
|
// row 0: price=150, type=0(A) → passes both
|
||||||
// row 1: price=150, type=2(C) → fails enum only
|
// row 1: price=150, type=2(C) → fails enum only
|
||||||
// row 2: price=600, type=0(A) → fails numeric only
|
// row 2: price=600, type=0(A) → fails numeric only
|
||||||
|
// row 3: price=600, type=2(C) → fails both
|
||||||
let tq = test_quant(2, 1);
|
let tq = test_quant(2, 1);
|
||||||
let feature_data = vec![
|
let feature_data = vec![
|
||||||
tq.encode(0, 150.0),
|
tq.encode(0, 150.0),
|
||||||
|
|
@ -869,6 +866,8 @@ mod tests {
|
||||||
2u16, // row 1
|
2u16, // row 1
|
||||||
tq.encode(0, 600.0),
|
tq.encode(0, 600.0),
|
||||||
0u16, // row 2
|
0u16, // row 2
|
||||||
|
tq.encode(0, 600.0),
|
||||||
|
2u16, // row 3
|
||||||
];
|
];
|
||||||
let num_filters = vec![ParsedFilter {
|
let num_filters = vec![ParsedFilter {
|
||||||
feat_idx: 0,
|
feat_idx: 0,
|
||||||
|
|
@ -880,20 +879,20 @@ mod tests {
|
||||||
allowed: [0u16, 1].into_iter().collect(),
|
allowed: [0u16, 1].into_iter().collect(),
|
||||||
}];
|
}];
|
||||||
|
|
||||||
let (total, impacts) =
|
let (total, rejections) =
|
||||||
count_filter_impacts(&num_filters, &enum_filters, &feature_data, 2, 0..3u32);
|
count_filter_rejections(&num_filters, &enum_filters, &feature_data, 2, 0..4u32);
|
||||||
|
|
||||||
assert_eq!(total, 1); // row 0
|
assert_eq!(total, 1); // row 0
|
||||||
assert_eq!(impacts[0], 1); // row 2 fails numeric only → impacts[0]
|
assert_eq!(rejections[0], 2); // rows 2 and 3 fail numeric
|
||||||
assert_eq!(impacts[1], 1); // row 1 fails enum only → impacts[1]
|
assert_eq!(rejections[1], 2); // rows 1 and 3 fail enum
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn filter_impacts_no_filters() {
|
fn filter_rejections_no_filters() {
|
||||||
let tq = test_quant(1, 1);
|
let tq = test_quant(1, 1);
|
||||||
let feature_data = vec![tq.encode(0, 100.0)];
|
let feature_data = vec![tq.encode(0, 100.0)];
|
||||||
let (total, impacts) = count_filter_impacts(&[], &[], &feature_data, 1, 0..1u32);
|
let (total, rejections) = count_filter_rejections(&[], &[], &feature_data, 1, 0..1u32);
|
||||||
assert_eq!(total, 1);
|
assert_eq!(total, 1);
|
||||||
assert!(impacts.is_empty());
|
assert!(rejections.is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ mod postcode_stats;
|
||||||
mod postcodes;
|
mod postcodes;
|
||||||
pub(crate) mod pricing;
|
pub(crate) mod pricing;
|
||||||
pub(crate) mod properties;
|
pub(crate) mod properties;
|
||||||
|
mod rightmove;
|
||||||
mod screenshot;
|
mod screenshot;
|
||||||
mod shorten;
|
mod shorten;
|
||||||
mod stats;
|
mod stats;
|
||||||
|
|
@ -47,6 +48,7 @@ pub use postcode_stats::get_postcode_stats;
|
||||||
pub use postcodes::{get_nearest_postcode, get_postcode_lookup, get_postcodes};
|
pub use postcodes::{get_nearest_postcode, get_postcode_lookup, get_postcodes};
|
||||||
pub use pricing::get_pricing;
|
pub use pricing::get_pricing;
|
||||||
pub use properties::get_hexagon_properties;
|
pub use properties::get_hexagon_properties;
|
||||||
|
pub use rightmove::get_rightmove_redirect;
|
||||||
pub use screenshot::{fetch_screenshot_bytes, get_screenshot};
|
pub use screenshot::{fetch_screenshot_bytes, get_screenshot};
|
||||||
pub use shorten::{get_short_url, post_shorten};
|
pub use shorten::{get_short_url, post_shorten};
|
||||||
pub use streetview::get_streetview;
|
pub use streetview::get_streetview;
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ use axum::response::Json;
|
||||||
use axum::Extension;
|
use axum::Extension;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Map, Value};
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::auth::OptionalUser;
|
use crate::auth::OptionalUser;
|
||||||
|
|
@ -60,7 +60,7 @@ pub struct AiFiltersResponse {
|
||||||
/// What the LLM couldn't map to existing filters (empty if everything matched)
|
/// What the LLM couldn't map to existing filters (empty if everything matched)
|
||||||
#[serde(skip_serializing_if = "String::is_empty")]
|
#[serde(skip_serializing_if = "String::is_empty")]
|
||||||
notes: String,
|
notes: String,
|
||||||
/// Number of properties matching the proposed filters (excludes travel time)
|
/// Number of properties matching the proposed property and travel time filters.
|
||||||
match_count: usize,
|
match_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -85,6 +85,77 @@ fn strip_markdown_fences(text: &str) -> &str {
|
||||||
trimmed
|
trimmed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn school_feature_name_from_key(name: &str) -> Option<&'static str> {
|
||||||
|
let rest = name.strip_prefix("Schools:")?;
|
||||||
|
let mut parts = rest.split(':');
|
||||||
|
let phase = parts.next()?;
|
||||||
|
let rating = parts.next()?;
|
||||||
|
let distance = parts.next()?;
|
||||||
|
|
||||||
|
match (phase, rating, distance) {
|
||||||
|
("primary", "good", "2") => Some("Good+ primary schools within 2km"),
|
||||||
|
("secondary", "good", "2") => Some("Good+ secondary schools within 2km"),
|
||||||
|
("primary", "outstanding", "2") => Some("Outstanding primary schools within 2km"),
|
||||||
|
("secondary", "outstanding", "2") => Some("Outstanding secondary schools within 2km"),
|
||||||
|
("primary", "good", "5") => Some("Good+ primary schools within 5km"),
|
||||||
|
("secondary", "good", "5") => Some("Good+ secondary schools within 5km"),
|
||||||
|
("primary", "outstanding", "5") => Some("Outstanding primary schools within 5km"),
|
||||||
|
("secondary", "outstanding", "5") => Some("Outstanding secondary schools within 5km"),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_synthetic_feature_key(name: &str, prefix: &str) -> Option<String> {
|
||||||
|
let rest = name.strip_prefix(prefix)?;
|
||||||
|
let (encoded, _id) = rest.rsplit_once(':')?;
|
||||||
|
urlencoding::decode(encoded)
|
||||||
|
.ok()
|
||||||
|
.map(|decoded| decoded.into_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert frontend synthetic filter keys back to backend feature names.
|
||||||
|
///
|
||||||
|
/// The React filter UI stores configurable cards under keys such as
|
||||||
|
/// `Political vote share:%25%20Labour:0`. The LLM and backend validators need
|
||||||
|
/// the real feature name (`% Labour`) instead.
|
||||||
|
fn backend_filter_name(name: &str) -> Option<String> {
|
||||||
|
if let Some(feature_name) = school_feature_name_from_key(name) {
|
||||||
|
return Some(feature_name.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
for prefix in [
|
||||||
|
"Specific crimes:",
|
||||||
|
"Political vote share:",
|
||||||
|
"Ethnicities:",
|
||||||
|
"Amenity distance:",
|
||||||
|
"Transport distance:",
|
||||||
|
"Amenities within 2km:",
|
||||||
|
"Amenities within 5km:",
|
||||||
|
] {
|
||||||
|
if let Some(feature_name) = decode_synthetic_feature_key(name, prefix) {
|
||||||
|
return Some(feature_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonical_filter_name(name: &str) -> String {
|
||||||
|
backend_filter_name(name).unwrap_or_else(|| name.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_context_filters(filters: &Value) -> Value {
|
||||||
|
let Some(obj) = filters.as_object() else {
|
||||||
|
return filters.clone();
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut normalized = Map::with_capacity(obj.len());
|
||||||
|
for (name, value) in obj {
|
||||||
|
normalized.insert(canonical_filter_name(name), value.clone());
|
||||||
|
}
|
||||||
|
Value::Object(normalized)
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the Gemini tool declaration for destination search.
|
/// Build the Gemini tool declaration for destination search.
|
||||||
fn build_tool_declarations(state: &AppState) -> Value {
|
fn build_tool_declarations(state: &AppState) -> Value {
|
||||||
let modes: Vec<&str> = state
|
let modes: Vec<&str> = state
|
||||||
|
|
@ -289,7 +360,7 @@ pub fn build_system_prompt(
|
||||||
- Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
|
- Use EXACT feature names from the list — spelling, capitalisation, and punctuation must match.\n\
|
||||||
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
|
- \"cheap\" / \"affordable\" = lower price range. \"expensive\" = higher price range.\n\
|
||||||
- \"low crime\" / \"safe\" = low values on Serious crime and Minor crime summary features. \
|
- \"low crime\" / \"safe\" = low values on Serious crime and Minor crime summary features. \
|
||||||
\"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of parks within 1km.\n\
|
\"quiet\" = low Noise (dB). \"green\" / \"near parks\" = high Number of amenities (Park) within 2km.\n\
|
||||||
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
|
- \"good schools\" = Good+ school features. \"outstanding schools\" = Outstanding school features.\n\
|
||||||
- When the user says a number like \"under 400k\", interpret it as 400000.\n\
|
- When the user says a number like \"under 400k\", interpret it as 400000.\n\
|
||||||
- When the user says \"3 bed\" or \"3 bedroom\", use Number of bedrooms & living rooms \
|
- When the user says \"3 bed\" or \"3 bedroom\", use Number of bedrooms & living rooms \
|
||||||
|
|
@ -429,7 +500,7 @@ pub fn build_system_prompt(
|
||||||
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
|
{\"name\": \"Noise (dB)\", \"bound\": \"max\", \"value\": 55}, \
|
||||||
{\"name\": \"Good+ primary schools within 2km\", \"bound\": \"min\", \"value\": 2}, \
|
{\"name\": \"Good+ primary schools within 2km\", \"bound\": \"min\", \"value\": 2}, \
|
||||||
{\"name\": \"Good+ secondary schools within 2km\", \"bound\": \"min\", \"value\": 1}, \
|
{\"name\": \"Good+ secondary schools within 2km\", \"bound\": \"min\", \"value\": 1}, \
|
||||||
{\"name\": \"Number of parks within 1km\", \"bound\": \"min\", \"value\": 3}], \
|
{\"name\": \"Number of amenities (Park) within 2km\", \"bound\": \"min\", \"value\": 3}], \
|
||||||
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
|
\"enum_filters\": [], \"travel_time_filters\": [], \"notes\": \"\"}"
|
||||||
.to_string(),
|
.to_string(),
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,9 @@ use crate::data::{Histogram, PropertyData};
|
||||||
use crate::features::{self, Feature, FEATURE_GROUPS};
|
use crate::features::{self, Feature, FEATURE_GROUPS};
|
||||||
use crate::state::SharedState;
|
use crate::state::SharedState;
|
||||||
|
|
||||||
|
const FILTER_GROUP_ORDER: &[&str] = &["Transport", "Property prices", "Properties", "Amenities"];
|
||||||
|
const LAST_FILTER_GROUPS: &[&str] = &["Area development"];
|
||||||
|
|
||||||
fn is_empty(val: &str) -> bool {
|
fn is_empty(val: &str) -> bool {
|
||||||
val.is_empty()
|
val.is_empty()
|
||||||
}
|
}
|
||||||
|
|
@ -62,6 +65,23 @@ pub struct FeaturesResponse {
|
||||||
pub groups: Vec<FeatureGroupResponse>,
|
pub groups: Vec<FeatureGroupResponse>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn filter_group_rank(name: &str) -> usize {
|
||||||
|
if let Some(index) = FILTER_GROUP_ORDER
|
||||||
|
.iter()
|
||||||
|
.position(|group_name| *group_name == name)
|
||||||
|
{
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
if LAST_FILTER_GROUPS.contains(&name) {
|
||||||
|
return usize::MAX;
|
||||||
|
}
|
||||||
|
FILTER_GROUP_ORDER.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn order_filter_groups(groups: &mut [FeatureGroupResponse]) {
|
||||||
|
groups.sort_by_key(|group| filter_group_rank(&group.name));
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the features response at startup. Called once and cached in AppState.
|
/// Build the features response at startup. Called once and cached in AppState.
|
||||||
/// Feature order in each group follows the array order in FEATURE_GROUPS.
|
/// Feature order in each group follows the array order in FEATURE_GROUPS.
|
||||||
pub fn build_features_response(data: &PropertyData) -> FeaturesResponse {
|
pub fn build_features_response(data: &PropertyData) -> FeaturesResponse {
|
||||||
|
|
@ -146,9 +166,9 @@ pub fn build_features_response(data: &PropertyData) -> FeaturesResponse {
|
||||||
max: stats.slider_max,
|
max: stats.slider_max,
|
||||||
step: 0.1,
|
step: 0.1,
|
||||||
histogram: stats.histogram.clone(),
|
histogram: stats.histogram.clone(),
|
||||||
description: format!("Distance to the closest {category} POI"),
|
description: format!("Distance to the closest {category} amenity"),
|
||||||
detail: format!(
|
detail: format!(
|
||||||
"Straight-line distance in kilometres from the postcode to the nearest {category} point of interest in the POI dataset."
|
"Straight-line distance in kilometres from the postcode to the nearest {category} amenity in the amenities dataset."
|
||||||
),
|
),
|
||||||
source: "osm-pois".to_string(),
|
source: "osm-pois".to_string(),
|
||||||
prefix: "",
|
prefix: "",
|
||||||
|
|
@ -159,17 +179,32 @@ pub fn build_features_response(data: &PropertyData) -> FeaturesResponse {
|
||||||
} else if let Some(category) = features::dynamic_poi_count_category(name) {
|
} else if let Some(category) = features::dynamic_poi_count_category(name) {
|
||||||
let stats = &data.poi_metrics.feature_stats[feat_idx];
|
let stats = &data.poi_metrics.feature_stats[feat_idx];
|
||||||
let radius = features::dynamic_poi_count_radius(name).unwrap_or(0);
|
let radius = features::dynamic_poi_count_radius(name).unwrap_or(0);
|
||||||
|
let is_park = category.eq_ignore_ascii_case("park");
|
||||||
dynamic_poi_features.push(FeatureInfo::Numeric {
|
dynamic_poi_features.push(FeatureInfo::Numeric {
|
||||||
name: name.clone(),
|
name: name.clone(),
|
||||||
min: stats.slider_min,
|
min: stats.slider_min,
|
||||||
max: stats.slider_max,
|
max: stats.slider_max,
|
||||||
step: 1.0,
|
step: 1.0,
|
||||||
histogram: stats.histogram.clone(),
|
histogram: stats.histogram.clone(),
|
||||||
description: format!("Number of {category} POIs within {radius}km"),
|
description: if is_park {
|
||||||
detail: format!(
|
format!("Number of parks and green spaces within {radius}km")
|
||||||
"Count of {category} points of interest within a {radius}km radius of the property's postcode centroid."
|
} else {
|
||||||
),
|
format!("Number of {category} amenities within {radius}km")
|
||||||
source: "osm-pois".to_string(),
|
},
|
||||||
|
detail: if is_park {
|
||||||
|
format!(
|
||||||
|
"Count of public parks, gardens, playing fields, and play spaces with at least one entrance within a {radius}km radius of the property's postcode centroid. Derived from the OS Open Greenspace dataset."
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"Count of {category} amenities within a {radius}km radius of the property's postcode centroid."
|
||||||
|
)
|
||||||
|
},
|
||||||
|
source: if is_park {
|
||||||
|
"os-open-greenspace".to_string()
|
||||||
|
} else {
|
||||||
|
"osm-pois".to_string()
|
||||||
|
},
|
||||||
prefix: "",
|
prefix: "",
|
||||||
suffix: "",
|
suffix: "",
|
||||||
raw: false,
|
raw: false,
|
||||||
|
|
@ -182,11 +217,17 @@ pub fn build_features_response(data: &PropertyData) -> FeaturesResponse {
|
||||||
FeatureInfo::Numeric { name, .. } => features::dynamic_poi_feature_sort_key(name),
|
FeatureInfo::Numeric { name, .. } => features::dynamic_poi_feature_sort_key(name),
|
||||||
FeatureInfo::Enum { name, .. } => features::dynamic_poi_feature_sort_key(name),
|
FeatureInfo::Enum { name, .. } => features::dynamic_poi_feature_sort_key(name),
|
||||||
});
|
});
|
||||||
|
if let Some(amenities_group) = groups.iter_mut().find(|group| group.name == "Amenities") {
|
||||||
|
amenities_group.features.extend(dynamic_poi_features);
|
||||||
|
} else {
|
||||||
groups.push(FeatureGroupResponse {
|
groups.push(FeatureGroupResponse {
|
||||||
name: "Nearby POIs".to_string(),
|
name: "Amenities".to_string(),
|
||||||
features: dynamic_poi_features,
|
features: dynamic_poi_features,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
order_filter_groups(&mut groups);
|
||||||
|
|
||||||
FeaturesResponse { groups }
|
FeaturesResponse { groups }
|
||||||
}
|
}
|
||||||
|
|
@ -196,3 +237,46 @@ pub async fn get_features(State(shared): State<Arc<SharedState>>) -> Json<Featur
|
||||||
info!("GET /api/features");
|
info!("GET /api/features");
|
||||||
Json(state.features_response.clone())
|
Json(state.features_response.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn group(name: &str) -> FeatureGroupResponse {
|
||||||
|
FeatureGroupResponse {
|
||||||
|
name: name.to_string(),
|
||||||
|
features: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn orders_filter_groups_for_backend_response() {
|
||||||
|
let mut groups = vec![
|
||||||
|
group("Properties"),
|
||||||
|
group("Education"),
|
||||||
|
group("Area development"),
|
||||||
|
group("Property prices"),
|
||||||
|
group("Crime"),
|
||||||
|
group("Neighbours"),
|
||||||
|
group("Amenities"),
|
||||||
|
group("Transport"),
|
||||||
|
];
|
||||||
|
|
||||||
|
order_filter_groups(&mut groups);
|
||||||
|
|
||||||
|
let names: Vec<&str> = groups.iter().map(|group| group.name.as_str()).collect();
|
||||||
|
assert_eq!(
|
||||||
|
names,
|
||||||
|
vec![
|
||||||
|
"Transport",
|
||||||
|
"Property prices",
|
||||||
|
"Properties",
|
||||||
|
"Amenities",
|
||||||
|
"Education",
|
||||||
|
"Crime",
|
||||||
|
"Neighbours",
|
||||||
|
"Area development",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -68,7 +68,6 @@ pub async fn get_filter_counts(
|
||||||
}
|
}
|
||||||
|
|
||||||
let filters_str = params.filters;
|
let filters_str = params.filters;
|
||||||
let has_poi_filters = !parsed_poi_filters.is_empty();
|
|
||||||
|
|
||||||
let response = tokio::task::spawn_blocking(move || -> Result<FilterCountsResponse, String> {
|
let response = tokio::task::spawn_blocking(move || -> Result<FilterCountsResponse, String> {
|
||||||
let t0 = std::time::Instant::now();
|
let t0 = std::time::Instant::now();
|
||||||
|
|
@ -99,54 +98,40 @@ pub async fn get_filter_counts(
|
||||||
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
||||||
let row = row_idx as usize;
|
let row = row_idx as usize;
|
||||||
let base = row * num_features;
|
let base = row * num_features;
|
||||||
let mut fail_count: u32 = 0;
|
let mut passes_all = true;
|
||||||
let mut fail_index: usize = 0;
|
|
||||||
|
|
||||||
// Test numeric filters
|
// Test numeric filters
|
||||||
for (i, f) in parsed_filters.iter().enumerate() {
|
for (i, f) in parsed_filters.iter().enumerate() {
|
||||||
let raw = feature_data[base + f.feat_idx];
|
let raw = feature_data[base + f.feat_idx];
|
||||||
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
||||||
fail_count += 1;
|
impacts[i] += 1;
|
||||||
fail_index = i;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test enum filters
|
// Test enum filters
|
||||||
if fail_count <= 1 {
|
|
||||||
for (i, f) in parsed_enum_filters.iter().enumerate() {
|
for (i, f) in parsed_enum_filters.iter().enumerate() {
|
||||||
let raw = feature_data[base + f.feat_idx];
|
let raw = feature_data[base + f.feat_idx];
|
||||||
if raw == NAN_U16 || !f.allowed.contains(&raw) {
|
if raw == NAN_U16 || !f.allowed.contains(&raw) {
|
||||||
fail_count += 1;
|
impacts[parsed_filters.len() + i] += 1;
|
||||||
fail_index = parsed_filters.len() + i;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test travel time filters
|
// Test POI filters
|
||||||
if fail_count <= 1 && has_poi_filters {
|
|
||||||
for (i, f) in parsed_poi_filters.iter().enumerate() {
|
for (i, f) in parsed_poi_filters.iter().enumerate() {
|
||||||
let raw = state
|
let raw = state
|
||||||
.data
|
.data
|
||||||
.poi_metrics
|
.poi_metrics
|
||||||
.raw_for_property_row(row, f.metric_idx);
|
.raw_for_property_row(row, f.metric_idx);
|
||||||
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
if raw == NAN_U16 || raw < f.min_u16 || raw > f.max_u16 {
|
||||||
fail_count += 1;
|
impacts[parsed_filters.len() + parsed_enum_filters.len() + i] += 1;
|
||||||
fail_index = parsed_filters.len() + parsed_enum_filters.len() + i;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test travel time filters
|
// Test travel time filters
|
||||||
if fail_count <= 1 && has_travel {
|
if has_travel {
|
||||||
let postcode = pc_interner.resolve(&pc_keys[row]);
|
let postcode = pc_interner.resolve(&pc_keys[row]);
|
||||||
for (slot, &ti) in travel_filter_indices.iter().enumerate() {
|
for (slot, &ti) in travel_filter_indices.iter().enumerate() {
|
||||||
let entry = &travel_entries[ti];
|
let entry = &travel_entries[ti];
|
||||||
|
|
@ -165,19 +150,14 @@ pub async fn get_filter_counts(
|
||||||
_ => true,
|
_ => true,
|
||||||
};
|
};
|
||||||
if !passes {
|
if !passes {
|
||||||
fail_count += 1;
|
impacts[num_regular + slot] += 1;
|
||||||
fail_index = num_regular + slot;
|
passes_all = false;
|
||||||
if fail_count > 1 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match fail_count {
|
if passes_all {
|
||||||
0 => total_passing += 1,
|
total_passing += 1;
|
||||||
1 => impacts[fail_index] += 1,
|
|
||||||
_ => {}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ use axum::response::{IntoResponse, Json};
|
||||||
use axum::Extension;
|
use axum::Extension;
|
||||||
use metrics::histogram;
|
use metrics::histogram;
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use rustc_hash::FxHashMap;
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
@ -32,6 +32,7 @@ type ChunkResult = (
|
||||||
FxHashMap<u64, Aggregator>,
|
FxHashMap<u64, Aggregator>,
|
||||||
FxHashMap<u64, PoiAggregator>,
|
FxHashMap<u64, PoiAggregator>,
|
||||||
Vec<FxHashMap<u64, TravelTimeAgg>>,
|
Vec<FxHashMap<u64, TravelTimeAgg>>,
|
||||||
|
FxHashSet<u64>,
|
||||||
);
|
);
|
||||||
|
|
||||||
/// Maximum center-to-vertex distance in degrees per H3 resolution.
|
/// Maximum center-to-vertex distance in degrees per H3 resolution.
|
||||||
|
|
@ -82,6 +83,7 @@ pub struct HexagonParams {
|
||||||
fn build_feature_maps(
|
fn build_feature_maps(
|
||||||
groups: &FxHashMap<u64, Aggregator>,
|
groups: &FxHashMap<u64, Aggregator>,
|
||||||
poi_groups: &FxHashMap<u64, PoiAggregator>,
|
poi_groups: &FxHashMap<u64, PoiAggregator>,
|
||||||
|
selectable_cells: &FxHashSet<u64>,
|
||||||
min_keys: &[String],
|
min_keys: &[String],
|
||||||
max_keys: &[String],
|
max_keys: &[String],
|
||||||
avg_keys: &[String],
|
avg_keys: &[String],
|
||||||
|
|
@ -214,6 +216,36 @@ fn build_feature_maps(
|
||||||
features.push(map);
|
features.push(map);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for &cell_id in selectable_cells {
|
||||||
|
if groups.contains_key(&cell_id) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(cell) = h3o::CellIndex::try_from(cell_id).ok() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let center: h3o::LatLng = cell.into();
|
||||||
|
let lat = center.lat();
|
||||||
|
let lng = center.lng();
|
||||||
|
|
||||||
|
if lat < bound_south || lat > bound_north || lng < bound_west || lng > bound_east {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut map = Map::new();
|
||||||
|
map.insert("h3".into(), Value::String(cell.to_string()));
|
||||||
|
map.insert("count".into(), Value::from(0));
|
||||||
|
if let (Some(lat_num), Some(lon_num)) = (
|
||||||
|
serde_json::Number::from_f64(lat),
|
||||||
|
serde_json::Number::from_f64(lng),
|
||||||
|
) {
|
||||||
|
map.insert("lat".into(), Value::Number(lat_num));
|
||||||
|
map.insert("lon".into(), Value::Number(lon_num));
|
||||||
|
}
|
||||||
|
features.push(map);
|
||||||
|
}
|
||||||
|
|
||||||
features
|
features
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -313,6 +345,7 @@ pub async fn get_hexagons(
|
||||||
|
|
||||||
let mut groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
|
let mut groups: FxHashMap<u64, Aggregator> = FxHashMap::default();
|
||||||
let mut poi_groups: FxHashMap<u64, PoiAggregator> = FxHashMap::default();
|
let mut poi_groups: FxHashMap<u64, PoiAggregator> = FxHashMap::default();
|
||||||
|
let mut selectable_cells: FxHashSet<u64> = FxHashSet::default();
|
||||||
let mut travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> = (0..travel_entries.len())
|
let mut travel_aggs: Vec<FxHashMap<u64, TravelTimeAgg>> = (0..travel_entries.len())
|
||||||
.map(|_| FxHashMap::default())
|
.map(|_| FxHashMap::default())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
@ -338,12 +371,22 @@ pub async fn get_hexagons(
|
||||||
..travel_entries.len())
|
..travel_entries.len())
|
||||||
.map(|_| FxHashMap::default())
|
.map(|_| FxHashMap::default())
|
||||||
.collect();
|
.collect();
|
||||||
|
let mut local_selectable_cells: FxHashSet<u64> = FxHashSet::default();
|
||||||
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
|
let mut h3_cache: FxHashMap<u64, u64> = FxHashMap::default();
|
||||||
let mut travel_minutes: Vec<Option<i16>> =
|
let mut travel_minutes: Vec<Option<i16>> =
|
||||||
Vec::with_capacity(travel_entries.len());
|
Vec::with_capacity(travel_entries.len());
|
||||||
|
|
||||||
'row: for &row_idx in chunk {
|
'row: for &row_idx in chunk {
|
||||||
let row = row_idx as usize;
|
let row = row_idx as usize;
|
||||||
|
let cell_id = cell_for_row_cached(
|
||||||
|
row,
|
||||||
|
precomputed,
|
||||||
|
h3_res,
|
||||||
|
need_parent,
|
||||||
|
&mut h3_cache,
|
||||||
|
);
|
||||||
|
local_selectable_cells.insert(cell_id);
|
||||||
|
|
||||||
if !row_passes_filters(
|
if !row_passes_filters(
|
||||||
row,
|
row,
|
||||||
&parsed_filters,
|
&parsed_filters,
|
||||||
|
|
@ -384,14 +427,6 @@ pub async fn get_hexagons(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let cell_id = cell_for_row_cached(
|
|
||||||
row,
|
|
||||||
precomputed,
|
|
||||||
h3_res,
|
|
||||||
need_parent,
|
|
||||||
&mut h3_cache,
|
|
||||||
);
|
|
||||||
|
|
||||||
let agg = local_groups
|
let agg = local_groups
|
||||||
.entry(cell_id)
|
.entry(cell_id)
|
||||||
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
||||||
|
|
@ -424,12 +459,19 @@ pub async fn get_hexagons(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
(local_groups, local_poi_groups, local_travel_aggs)
|
(
|
||||||
|
local_groups,
|
||||||
|
local_poi_groups,
|
||||||
|
local_travel_aggs,
|
||||||
|
local_selectable_cells,
|
||||||
|
)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Merge thread-local results into the main accumulators
|
// Merge thread-local results into the main accumulators
|
||||||
for (local_groups, local_poi_groups, local_travel) in thread_results {
|
for (local_groups, local_poi_groups, local_travel, local_selectable_cells) in
|
||||||
|
thread_results
|
||||||
|
{
|
||||||
for (cell_id, local_agg) in local_groups {
|
for (cell_id, local_agg) in local_groups {
|
||||||
groups
|
groups
|
||||||
.entry(cell_id)
|
.entry(cell_id)
|
||||||
|
|
@ -450,6 +492,7 @@ pub async fn get_hexagons(
|
||||||
.merge(&local_tt);
|
.merge(&local_tt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
selectable_cells.extend(local_selectable_cells);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Sequential: use for_each_in_bounds to avoid Vec<u32> allocation
|
// Sequential: use for_each_in_bounds to avoid Vec<u32> allocation
|
||||||
|
|
@ -460,6 +503,9 @@ pub async fn get_hexagons(
|
||||||
.grid
|
.grid
|
||||||
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
||||||
let row = row_idx as usize;
|
let row = row_idx as usize;
|
||||||
|
let cell_id =
|
||||||
|
cell_for_row_cached(row, precomputed, h3_res, need_parent, &mut h3_cache);
|
||||||
|
selectable_cells.insert(cell_id);
|
||||||
|
|
||||||
if !row_passes_filters(
|
if !row_passes_filters(
|
||||||
row,
|
row,
|
||||||
|
|
@ -499,9 +545,6 @@ pub async fn get_hexagons(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let cell_id =
|
|
||||||
cell_for_row_cached(row, precomputed, h3_res, need_parent, &mut h3_cache);
|
|
||||||
|
|
||||||
let aggregation = groups
|
let aggregation = groups
|
||||||
.entry(cell_id)
|
.entry(cell_id)
|
||||||
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
||||||
|
|
@ -540,6 +583,7 @@ pub async fn get_hexagons(
|
||||||
let mut features = build_feature_maps(
|
let mut features = build_feature_maps(
|
||||||
&groups,
|
&groups,
|
||||||
&poi_groups,
|
&poi_groups,
|
||||||
|
&selectable_cells,
|
||||||
min_keys,
|
min_keys,
|
||||||
max_keys,
|
max_keys,
|
||||||
avg_keys,
|
avg_keys,
|
||||||
|
|
@ -564,6 +608,7 @@ pub async fn get_hexagons(
|
||||||
resolution,
|
resolution,
|
||||||
rows = row_count,
|
rows = row_count,
|
||||||
parallel,
|
parallel,
|
||||||
|
selectable_cells = selectable_cells.len(),
|
||||||
cells_before_filter = groups.len(),
|
cells_before_filter = groups.len(),
|
||||||
cells_after_filter = features.len(),
|
cells_after_filter = features.len(),
|
||||||
truncated,
|
truncated,
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ use axum::http::StatusCode;
|
||||||
use axum::response::{IntoResponse, Json};
|
use axum::response::{IntoResponse, Json};
|
||||||
use axum::Extension;
|
use axum::Extension;
|
||||||
use metrics::histogram;
|
use metrics::histogram;
|
||||||
use rustc_hash::FxHashMap;
|
use rustc_hash::{FxHashMap, FxHashSet};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
@ -142,11 +142,18 @@ pub async fn get_postcodes(
|
||||||
// Single-pass: aggregate directly into postcode_aggs while iterating properties in bounds
|
// Single-pass: aggregate directly into postcode_aggs while iterating properties in bounds
|
||||||
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
|
let mut postcode_aggs: FxHashMap<usize, Aggregator> = FxHashMap::default();
|
||||||
let mut poi_aggs: FxHashMap<usize, PoiAggregator> = FxHashMap::default();
|
let mut poi_aggs: FxHashMap<usize, PoiAggregator> = FxHashMap::default();
|
||||||
|
let mut selectable_postcodes: FxHashSet<usize> = FxHashSet::default();
|
||||||
|
|
||||||
state
|
state
|
||||||
.grid
|
.grid
|
||||||
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
||||||
let row = row_idx as usize;
|
let row = row_idx as usize;
|
||||||
|
let postcode = state.data.postcode(row);
|
||||||
|
let Some(&pc_idx) = postcode_data.postcode_to_idx.get(postcode) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
selectable_postcodes.insert(pc_idx);
|
||||||
|
|
||||||
if !row_passes_filters(
|
if !row_passes_filters(
|
||||||
row,
|
row,
|
||||||
&parsed_filters,
|
&parsed_filters,
|
||||||
|
|
@ -161,8 +168,6 @@ pub async fn get_postcodes(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let postcode = state.data.postcode(row);
|
|
||||||
if let Some(&pc_idx) = postcode_data.postcode_to_idx.get(postcode) {
|
|
||||||
let agg = postcode_aggs
|
let agg = postcode_aggs
|
||||||
.entry(pc_idx)
|
.entry(pc_idx)
|
||||||
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
.or_insert_with(|| Aggregator::new(num_features, enum_dist_config));
|
||||||
|
|
@ -177,7 +182,6 @@ pub async fn get_postcodes(
|
||||||
.or_insert_with(|| PoiAggregator::new(poi_num_features))
|
.or_insert_with(|| PoiAggregator::new(poi_num_features))
|
||||||
.add_row_selective(poi_metrics, row, poi_field_indices);
|
.add_row_selective(poi_metrics, row, poi_field_indices);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// Filter postcodes by travel time range (if specified)
|
// Filter postcodes by travel time range (if specified)
|
||||||
|
|
@ -229,8 +233,10 @@ pub async fn get_postcodes(
|
||||||
let t_agg = t0.elapsed();
|
let t_agg = t0.elapsed();
|
||||||
|
|
||||||
// Build response, filtering postcodes to only those whose polygon intersects query bounds
|
// Build response, filtering postcodes to only those whose polygon intersects query bounds
|
||||||
let mut features = Vec::with_capacity(postcode_aggs.len());
|
let mut features = Vec::with_capacity(selectable_postcodes.len());
|
||||||
let postcodes_before_filter = postcode_aggs.len();
|
let postcodes_before_filter = selectable_postcodes.len();
|
||||||
|
let matching_postcodes = postcode_aggs.len();
|
||||||
|
let mut included_postcodes: FxHashSet<usize> = FxHashSet::default();
|
||||||
let mut filtered_out = 0usize;
|
let mut filtered_out = 0usize;
|
||||||
|
|
||||||
for (pc_idx, aggregation) in postcode_aggs {
|
for (pc_idx, aggregation) in postcode_aggs {
|
||||||
|
|
@ -255,7 +261,7 @@ pub async fn get_postcodes(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let geometry = postcode_data.geometries[pc_idx].clone();
|
let geometry = postcode_data.geometry_geojson(pc_idx);
|
||||||
|
|
||||||
// Build properties
|
// Build properties
|
||||||
let centroid = postcode_data.centroids[pc_idx];
|
let centroid = postcode_data.centroids[pc_idx];
|
||||||
|
|
@ -347,18 +353,71 @@ pub async fn get_postcodes(
|
||||||
feature.insert("properties".into(), Value::Object(props));
|
feature.insert("properties".into(), Value::Object(props));
|
||||||
|
|
||||||
features.push(feature);
|
features.push(feature);
|
||||||
|
included_postcodes.insert(pc_idx);
|
||||||
|
|
||||||
if features.len() >= MAX_CELLS_PER_REQUEST {
|
if features.len() >= MAX_CELLS_PER_REQUEST {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if features.len() < MAX_CELLS_PER_REQUEST {
|
||||||
|
for pc_idx in selectable_postcodes {
|
||||||
|
if included_postcodes.contains(&pc_idx) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let (pc_south, pc_west, pc_north, pc_east) = postcode_data.aabbs[pc_idx];
|
||||||
|
|
||||||
|
if !bounds_intersect(
|
||||||
|
pc_south as f64,
|
||||||
|
pc_west as f64,
|
||||||
|
pc_north as f64,
|
||||||
|
pc_east as f64,
|
||||||
|
south,
|
||||||
|
west,
|
||||||
|
north,
|
||||||
|
east,
|
||||||
|
) {
|
||||||
|
filtered_out += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let geometry = postcode_data.geometry_geojson(pc_idx);
|
||||||
|
let centroid = postcode_data.centroids[pc_idx];
|
||||||
|
let mut props = Map::new();
|
||||||
|
props.insert(
|
||||||
|
"postcode".into(),
|
||||||
|
Value::String(postcode_data.postcodes[pc_idx].clone()),
|
||||||
|
);
|
||||||
|
props.insert("count".into(), Value::from(0));
|
||||||
|
props.insert(
|
||||||
|
"centroid".into(),
|
||||||
|
Value::Array(vec![
|
||||||
|
Value::from(centroid.1 as f64),
|
||||||
|
Value::from(centroid.0 as f64),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut feature = Map::new();
|
||||||
|
feature.insert("type".into(), Value::String("Feature".into()));
|
||||||
|
feature.insert("geometry".into(), geometry);
|
||||||
|
feature.insert("properties".into(), Value::Object(props));
|
||||||
|
|
||||||
|
features.push(feature);
|
||||||
|
|
||||||
|
if features.len() >= MAX_CELLS_PER_REQUEST {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
histogram!("postcodes_response_count").record(features.len() as f64);
|
histogram!("postcodes_response_count").record(features.len() as f64);
|
||||||
|
|
||||||
let truncated = features.len() >= MAX_CELLS_PER_REQUEST;
|
let truncated = features.len() >= MAX_CELLS_PER_REQUEST;
|
||||||
let t_total = t0.elapsed();
|
let t_total = t0.elapsed();
|
||||||
info!(
|
info!(
|
||||||
postcodes_before_filter,
|
postcodes_before_filter,
|
||||||
|
matching_postcodes,
|
||||||
postcodes_after_filter = features.len(),
|
postcodes_after_filter = features.len(),
|
||||||
filtered_out,
|
filtered_out,
|
||||||
truncated,
|
truncated,
|
||||||
|
|
@ -418,7 +477,7 @@ pub async fn get_nearest_postcode(
|
||||||
|
|
||||||
let idx = best_idx.ok_or(StatusCode::NOT_FOUND)?;
|
let idx = best_idx.ok_or(StatusCode::NOT_FOUND)?;
|
||||||
let (lat, lon) = postcode_data.centroids[idx];
|
let (lat, lon) = postcode_data.centroids[idx];
|
||||||
let geometry = postcode_data.geometries[idx].clone();
|
let geometry = postcode_data.geometry_geojson(idx);
|
||||||
let postcode = &postcode_data.postcodes[idx];
|
let postcode = &postcode_data.postcodes[idx];
|
||||||
|
|
||||||
// Log location for authenticated users (best-effort, non-blocking)
|
// Log location for authenticated users (best-effort, non-blocking)
|
||||||
|
|
@ -454,7 +513,7 @@ pub async fn get_postcode_lookup(
|
||||||
|
|
||||||
if let Some(&idx) = postcode_data.postcode_to_idx.get(&normalized) {
|
if let Some(&idx) = postcode_data.postcode_to_idx.get(&normalized) {
|
||||||
let (lat, lon) = postcode_data.centroids[idx];
|
let (lat, lon) = postcode_data.centroids[idx];
|
||||||
let geometry = postcode_data.geometries[idx].clone();
|
let geometry = postcode_data.geometry_geojson(idx);
|
||||||
|
|
||||||
info!(postcode = %normalized, "GET /api/postcode/{postcode}");
|
info!(postcode = %normalized, "GET /api/postcode/{postcode}");
|
||||||
Ok(Json(serde_json::json!({
|
Ok(Json(serde_json::json!({
|
||||||
|
|
|
||||||
237
server-rs/src/routes/rightmove.rs
Normal file
237
server-rs/src/routes/rightmove.rs
Normal file
|
|
@ -0,0 +1,237 @@
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use axum::extract::{Query, State};
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::response::Redirect;
|
||||||
|
use reqwest::Url;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
use crate::state::SharedState;
|
||||||
|
use crate::utils::normalize_postcode;
|
||||||
|
|
||||||
|
const RIGHTMOVE_TYPEAHEAD_URL: &str = "https://los.rightmove.co.uk/typeahead";
|
||||||
|
const RIGHTMOVE_HOST: &str = "www.rightmove.co.uk";
|
||||||
|
const RIGHTMOVE_FIND_PATH: &str = "/property-for-sale/find.html";
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct RightmoveRedirectParams {
|
||||||
|
postcode: String,
|
||||||
|
target: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct RightmoveTypeaheadResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
matches: Vec<RightmoveTypeaheadMatch>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct RightmoveTypeaheadMatch {
|
||||||
|
id: Value,
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
match_type: String,
|
||||||
|
#[serde(default, rename = "displayName")]
|
||||||
|
display_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_rightmove_redirect(
|
||||||
|
State(shared): State<Arc<SharedState>>,
|
||||||
|
Query(params): Query<RightmoveRedirectParams>,
|
||||||
|
) -> Result<Redirect, (StatusCode, String)> {
|
||||||
|
if !looks_like_full_uk_postcode(¶ms.postcode) {
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
"'postcode' must be a full UK postcode".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let postcode = normalize_postcode(¶ms.postcode);
|
||||||
|
let mut target = parse_rightmove_target(¶ms.target)?;
|
||||||
|
let state = shared.load_state();
|
||||||
|
|
||||||
|
match fetch_exact_postcode_location_identifier(&state.http_client, &postcode).await {
|
||||||
|
Some(location_identifier) => {
|
||||||
|
apply_exact_postcode_location(&mut target, &postcode, &location_identifier);
|
||||||
|
}
|
||||||
|
None => warn!(
|
||||||
|
postcode,
|
||||||
|
"Could not resolve exact Rightmove postcode location"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Redirect::temporary(target.as_str()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_exact_postcode_location_identifier(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
postcode: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
let url = format!(
|
||||||
|
"{}?query={}&limit=5",
|
||||||
|
RIGHTMOVE_TYPEAHEAD_URL,
|
||||||
|
urlencoding::encode(postcode)
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = client
|
||||||
|
.get(url)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|err| warn!(postcode, "Rightmove typeahead request failed: {err}"))
|
||||||
|
.ok()?;
|
||||||
|
|
||||||
|
if !response.status().is_success() {
|
||||||
|
warn!(
|
||||||
|
postcode,
|
||||||
|
status = %response.status(),
|
||||||
|
"Rightmove typeahead returned an error"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let typeahead: RightmoveTypeaheadResponse = response
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|err| {
|
||||||
|
warn!(
|
||||||
|
postcode,
|
||||||
|
"Failed to parse Rightmove typeahead response: {err}"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.ok()?;
|
||||||
|
|
||||||
|
typeahead.matches.iter().find_map(|item| {
|
||||||
|
if !item.match_type.eq_ignore_ascii_case("POSTCODE") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if compact_postcode(&item.display_name) != compact_postcode(postcode) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
rightmove_id_to_string(&item.id).map(|id| format!("POSTCODE^{id}"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_rightmove_target(target: &str) -> Result<Url, (StatusCode, String)> {
|
||||||
|
let url = Url::parse(target).map_err(|_| {
|
||||||
|
(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
"'target' must be a valid Rightmove URL".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if url.scheme() != "https"
|
||||||
|
|| url.host_str() != Some(RIGHTMOVE_HOST)
|
||||||
|
|| url.path() != RIGHTMOVE_FIND_PATH
|
||||||
|
{
|
||||||
|
return Err((
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
"'target' must be a Rightmove property search URL".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_exact_postcode_location(url: &mut Url, postcode: &str, location_identifier: &str) {
|
||||||
|
let mut pairs: Vec<(String, String)> = url
|
||||||
|
.query_pairs()
|
||||||
|
.filter(|(key, _)| {
|
||||||
|
key != "searchLocation"
|
||||||
|
&& key != "useLocationIdentifier"
|
||||||
|
&& key != "locationIdentifier"
|
||||||
|
&& key != "radius"
|
||||||
|
})
|
||||||
|
.map(|(key, value)| (key.into_owned(), value.into_owned()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
pairs.push(("searchLocation".to_string(), postcode.to_string()));
|
||||||
|
pairs.push(("useLocationIdentifier".to_string(), "true".to_string()));
|
||||||
|
pairs.push((
|
||||||
|
"locationIdentifier".to_string(),
|
||||||
|
location_identifier.to_string(),
|
||||||
|
));
|
||||||
|
pairs.push(("radius".to_string(), "0.0".to_string()));
|
||||||
|
|
||||||
|
let mut query = url.query_pairs_mut();
|
||||||
|
query.clear();
|
||||||
|
for (key, value) in pairs {
|
||||||
|
query.append_pair(&key, &value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rightmove_id_to_string(value: &Value) -> Option<String> {
|
||||||
|
match value {
|
||||||
|
Value::String(id) if !id.trim().is_empty() => Some(id.clone()),
|
||||||
|
Value::Number(id) => Some(id.to_string()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compact_postcode(postcode: &str) -> String {
|
||||||
|
postcode
|
||||||
|
.chars()
|
||||||
|
.filter(|ch| !ch.is_whitespace())
|
||||||
|
.map(|ch| ch.to_ascii_uppercase())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn looks_like_full_uk_postcode(postcode: &str) -> bool {
|
||||||
|
let compact = compact_postcode(postcode);
|
||||||
|
let bytes = compact.as_bytes();
|
||||||
|
if !(5..=7).contains(&bytes.len()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let outward_len = bytes.len() - 3;
|
||||||
|
bytes[0].is_ascii_alphabetic()
|
||||||
|
&& bytes[..outward_len]
|
||||||
|
.iter()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric())
|
||||||
|
&& bytes[outward_len].is_ascii_digit()
|
||||||
|
&& bytes[outward_len + 1].is_ascii_alphabetic()
|
||||||
|
&& bytes[outward_len + 2].is_ascii_alphabetic()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rewrites_rightmove_url_to_exact_postcode_location() {
|
||||||
|
let mut url = Url::parse(
|
||||||
|
"https://www.rightmove.co.uk/property-for-sale/find.html?searchLocation=SW1A+1AA&useLocationIdentifier=true&locationIdentifier=OUTCODE%5E2506&radius=0.25&minPrice=100000",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
apply_exact_postcode_location(&mut url, "SW1A 1AA", "POSTCODE^837246");
|
||||||
|
|
||||||
|
let pairs: std::collections::HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||||
|
assert_eq!(pairs.get("searchLocation").unwrap(), "SW1A 1AA");
|
||||||
|
assert_eq!(pairs.get("useLocationIdentifier").unwrap(), "true");
|
||||||
|
assert_eq!(pairs.get("locationIdentifier").unwrap(), "POSTCODE^837246");
|
||||||
|
assert_eq!(pairs.get("radius").unwrap(), "0.0");
|
||||||
|
assert_eq!(pairs.get("minPrice").unwrap(), "100000");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_non_rightmove_redirect_targets() {
|
||||||
|
assert!(parse_rightmove_target("https://example.com/property-for-sale/find.html").is_err());
|
||||||
|
assert!(
|
||||||
|
parse_rightmove_target("http://www.rightmove.co.uk/property-for-sale/find.html")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
parse_rightmove_target("https://www.rightmove.co.uk/property-to-rent/find.html")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validates_full_postcode_shape() {
|
||||||
|
assert!(looks_like_full_uk_postcode("SW1A 1AA"));
|
||||||
|
assert!(looks_like_full_uk_postcode("e16an"));
|
||||||
|
assert!(!looks_like_full_uk_postcode("SW1A"));
|
||||||
|
assert!(!looks_like_full_uk_postcode("not a postcode"));
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue