all is well
Some checks failed
Build and publish Docker image / build-and-push (push) Failing after 7m0s
CI / Check (push) Failing after 7m9s

This commit is contained in:
Andras Schmelczer 2026-05-17 17:20:19 +01:00
parent eac1bd0d13
commit 2f149503bb
53 changed files with 1543 additions and 354 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

View file

@ -82,7 +82,7 @@ PROPERTY_TYPE_MAP = {
"Farm / Barn": "Other",
"Farm House": "Other",
"House": "Detached",
"House of Multiple Occupation": "Flats/Maisonettes",
"House of Multiple Occupation": "Other",
"House Share": "Other",
"Not Specified": "Other",
"Chalet": "Other",
@ -90,15 +90,15 @@ PROPERTY_TYPE_MAP = {
"Coach House": "Other",
"Character Property": "Other",
"Cluster House": "Other",
"Retirement Property": "Flats/Maisonettes",
"Retirement Property": "Other",
"Parking": "Other",
"Plot": "Other",
"Garages": "Other",
"Mews": "Terraced",
"Property": "Other",
"Flat Share": "Other",
"Block of Apartments": "Flats/Maisonettes",
"Private Halls": "Flats/Maisonettes",
"Block of Apartments": "Other",
"Private Halls": "Other",
"Terraced Bungalow": "Terraced",
"Equestrian Facility": "Other",
"Ground Maisonette": "Flats/Maisonettes",
@ -107,13 +107,13 @@ PROPERTY_TYPE_MAP = {
"Farm Land": "Other",
"House Boat": "Other",
"Barn": "Other",
"Serviced Apartments": "Flats/Maisonettes",
"Serviced Apartments": "Other",
# Space-separated variants (from home.co.uk underscore/hyphen normalization)
"Semi Detached": "Semi-Detached",
"Semi Detached Bungalow": "Semi-Detached",
"End Of Terrace": "Terraced",
"End Terrace": "Terraced",
"Block Of Apartments": "Flats/Maisonettes",
"Block Of Apartments": "Other",
# Lowercase variants (from home.co.uk / Rightmove APIs)
"house": "Detached",
"bungalow": "Other",
@ -121,7 +121,7 @@ PROPERTY_TYPE_MAP = {
"land": "Other",
"other": "Other",
"not-specified": "Other",
"retirement-property": "Flats/Maisonettes",
"retirement-property": "Other",
"equestrian-facility": "Other",
"flat": "Flats/Maisonettes",
"detached": "Detached",

View file

@ -19,7 +19,12 @@ from constants import (
RETRY_BASE_DELAY,
)
from spatial import PostcodeSpatialIndex
from transform import normalize_postcode, normalize_sub_type, validate_floor_area
from transform import (
normalize_postcode,
normalize_sub_type,
parse_int_value,
validate_floor_area,
)
log = logging.getLogger("homecouk")
@ -170,11 +175,19 @@ def parse_floor_area(description: str | None) -> float | None:
"""Try to extract floor area from description text like '789 sq.ft.' or '73 sq.m.'."""
if not description:
return None
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*ft", description, re.IGNORECASE)
m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))",
description,
re.IGNORECASE,
)
if m:
sqft = float(m.group(1).replace(",", ""))
return validate_floor_area(round(sqft * 0.092903, 1))
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*m", description, re.IGNORECASE)
m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))",
description,
re.IGNORECASE,
)
if m:
return validate_floor_area(round(float(m.group(1).replace(",", "")), 1))
return None
@ -237,6 +250,15 @@ def map_property_type(raw_type: str | None) -> str:
# Home.co.uk uses types like "House", "Flat", "Apartment", "Detached", etc.
# Try common patterns
lower = raw_type.lower()
excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if (
"flat" in lower
or "apartment" in lower
@ -269,8 +291,10 @@ def transform_property(
log.debug("Coords outside England: lat=%.4f lng=%.4f — skipping", lat, lng)
return None
price = prop.get("price") or prop.get("latest_price")
if not price or int(price) <= 0:
price = parse_int_value(prop.get("price")) or parse_int_value(
prop.get("latest_price")
)
if not price or price <= 0:
return None
# Home.co.uk provides postcodes directly, but fall back to spatial index
@ -281,10 +305,10 @@ def transform_property(
log.debug("No postcode for property at %.4f, %.4f — skipping", lat, lng)
return None
raw_beds = prop.get("bedrooms", 0) or 0
raw_baths = prop.get("bathrooms", 0) or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0
raw_beds = parse_int_value(prop.get("bedrooms")) or 0
raw_baths = parse_int_value(prop.get("bathrooms")) or 0
bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning(
"home.co.uk %s: implausible beds=%d baths=%d (capped to 0)",
@ -318,7 +342,7 @@ def transform_property(
"Leasehold/Freehold": parse_tenure(prop),
"Property type": map_property_type(listing_type),
"Property sub-type": normalize_sub_type(listing_type),
"price": int(price),
"price": price,
"price_frequency": "",
"Price qualifier": price_qualifier,
"Total floor area (sqm)": parse_floor_area(prop.get("description")),
@ -362,7 +386,16 @@ def search_outcode(
break
for prop in raw_props:
transformed = transform_property(prop, pc_index)
try:
transformed = transform_property(prop, pc_index)
except Exception as exc:
log.warning(
"home.co.uk %s property %s failed to transform: %s",
outcode,
prop.get("listing_id") or prop.get("property_id") or "?",
exc,
)
continue
if transformed:
properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties:

63
finder/listing_filters.py Normal file
View file

@ -0,0 +1,63 @@
"""Shared target filters for manual buy-listing scrapes."""
import math
from typing import Any
BUY_MAX_PRICE = 1_000_000
BUY_MIN_BEDROOMS = 2
BUY_MAX_BEDROOMS = 5
BUY_ALLOWED_BATHROOMS = frozenset({2, 3})
BUY_MIN_FLOOR_AREA_SQM = 90.0
BUY_MAX_FLOOR_AREA_SQM = 170.0
BUY_PROPERTY_TYPES = frozenset({"Flats/Maisonettes"})
BUY_MIN_FLOOR_AREA_SQFT = round(BUY_MIN_FLOOR_AREA_SQM / 0.092903)
BUY_MAX_FLOOR_AREA_SQFT = round(BUY_MAX_FLOOR_AREA_SQM / 0.092903)
def _number(value: Any) -> float | None:
if value is None:
return None
try:
number = float(value)
except (TypeError, ValueError):
return None
if not math.isfinite(number):
return None
return number
def _int(value: Any) -> int | None:
number = _number(value)
if number is None or not number.is_integer():
return None
return int(number)
def matches_strict_buy_listing_filter(prop: dict) -> bool:
"""Exact filter used to guard scraped/output datasets."""
if "price" in prop:
price = _number(prop.get("price"))
else:
price = _number(prop.get("Asking price"))
if price is None or price <= 0 or price >= BUY_MAX_PRICE:
return False
bedrooms = _int(prop.get("Bedrooms"))
if bedrooms is None or (
bedrooms < BUY_MIN_BEDROOMS or bedrooms > BUY_MAX_BEDROOMS
):
return False
property_type = prop.get("Property type")
if property_type not in BUY_PROPERTY_TYPES:
return False
bathrooms = _int(prop.get("Bathrooms"))
if bathrooms not in BUY_ALLOWED_BATHROOMS:
return False
floor_area = _number(prop.get("Total floor area (sqm)"))
if floor_area is None:
return False
return BUY_MIN_FLOOR_AREA_SQM <= floor_area <= BUY_MAX_FLOOR_AREA_SQM

View file

@ -10,6 +10,15 @@ from constants import (
TYPEAHEAD_URL,
)
from http_client import fetch_with_retry
from listing_filters import (
BUY_ALLOWED_BATHROOMS,
BUY_MAX_BEDROOMS,
BUY_MAX_FLOOR_AREA_SQFT,
BUY_MAX_PRICE,
BUY_MIN_BEDROOMS,
BUY_MIN_FLOOR_AREA_SQFT,
matches_strict_buy_listing_filter,
)
from spatial import PostcodeSpatialIndex
from transform import transform_property
@ -22,12 +31,23 @@ outcode_cache: dict[str, str] = {}
# Requesting index >= 1008 returns HTTP 400.
_MAX_INDEX = 1008
# Property type filters for splitting overcapped searches. Each sub-query
# gets its own 1008 cap, so we can recover listings beyond the unfiltered limit.
_PROPERTY_TYPES = [
"detached", "semi-detached", "terraced", "flat",
"bungalow", "park-home", "land",
]
_BASE_BUY_SEARCH_PARAMS = {
"propertyTypes": "flat",
"minBedrooms": str(BUY_MIN_BEDROOMS),
"maxBedrooms": str(BUY_MAX_BEDROOMS),
"minBathrooms": str(min(BUY_ALLOWED_BATHROOMS)),
"maxBathrooms": str(max(BUY_ALLOWED_BATHROOMS)),
"minSize": str(BUY_MIN_FLOOR_AREA_SQFT),
"maxSize": str(BUY_MAX_FLOOR_AREA_SQFT),
"maxPrice": str(BUY_MAX_PRICE - 1),
}
def _buy_search_params(extra_params: dict | None = None) -> dict:
params = dict(_BASE_BUY_SEARCH_PARAMS)
if extra_params:
params.update(extra_params)
return params
def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
@ -92,8 +112,18 @@ def _paginate(
break
for prop in raw_props:
transformed = transform_property(prop, outcode, pc_index)
if transformed:
try:
transformed = transform_property(prop, outcode, pc_index)
except Exception as exc:
log.warning(
"Rightmove %s/%s property %s failed to transform: %s",
outcode,
channel_cfg["channel"],
prop.get("id", "?"),
exc,
)
continue
if transformed and matches_strict_buy_listing_filter(transformed):
properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties:
return properties, result_count
@ -105,6 +135,15 @@ def _paginate(
if index >= result_count:
break
if index >= _MAX_INDEX:
log.warning(
"%s/%s: %d filtered results exceed Rightmove's %d-result page cap",
outcode,
channel_cfg["channel"],
result_count,
_MAX_INDEX,
)
break
time.sleep(DELAY_BETWEEN_PAGES)
@ -121,54 +160,20 @@ def search_outcode(
) -> list[dict]:
"""Paginate through search results for one outcode+channel. Returns transformed properties.
When the unfiltered result count exceeds 1008 (Rightmove's hard pagination cap),
re-queries per property type to recover listings beyond the cap.
Search requests set the supported Rightmove filters directly: flats,
2-5 bedrooms, 2-3 bathrooms, 969-1830 sq ft, and asking price below £1m.
"""
properties, result_count = _paginate(
client, outcode_id, outcode, channel_cfg, pc_index, max_properties=max_properties
properties, _ = _paginate(
client,
outcode_id,
outcode,
channel_cfg,
pc_index,
extra_params=_buy_search_params(),
max_properties=max_properties,
)
if max_properties is not None and len(properties) >= max_properties:
return properties[:max_properties]
if result_count <= _MAX_INDEX:
return properties
# Hit the 1008 cap — re-search per property type to get full coverage
ch = channel_cfg["channel"]
log.info(
"%s/%s: %d results exceed %d cap, splitting by property type",
outcode, ch, result_count, _MAX_INDEX,
)
all_by_id: dict[str, dict] = {p["id"]: p for p in properties}
for pt in _PROPERTY_TYPES:
pt_props, _ = _paginate(
client, outcode_id, outcode, channel_cfg, pc_index,
extra_params={"propertyTypes": pt},
max_properties=max_properties,
)
new = 0
for p in pt_props:
if p["id"] not in all_by_id:
all_by_id[p["id"]] = p
new += 1
if (
max_properties is not None
and len(all_by_id) >= max_properties
):
break
if new:
log.debug("%s/%s type=%s: +%d new properties", outcode, ch, pt, new)
if max_properties is not None and len(all_by_id) >= max_properties:
break
log.info(
"%s/%s: type split recovered %d%d properties",
outcode, ch, len(properties), len(all_by_id),
)
properties = list(all_by_id.values())
if max_properties is not None:
return properties[:max_properties]
return properties

View file

@ -19,6 +19,7 @@ from homecouk import load_cookies as load_homecouk_cookies
from homecouk import make_client as make_homecouk_client
from homecouk import search_outcode as homecouk_search_outcode
from http_client import make_client
from listing_filters import matches_strict_buy_listing_filter
from rightmove import resolve_outcode_id
from rightmove import search_outcode as rightmove_search_outcode
from spatial import PostcodeSpatialIndex
@ -181,11 +182,11 @@ def _source_names(sources: str | Iterable[str] | None) -> list[str]:
requested = [str(source).strip().lower() for source in sources]
requested = [source for source in requested if source]
if "all" in requested:
return list(SOURCE_ORDER)
unknown = sorted(set(requested) - set(SOURCE_ORDER))
unknown = sorted(set(requested) - set(SOURCE_ORDER) - {"all"})
if unknown:
raise ValueError(f"Unknown source(s): {', '.join(unknown)}")
if "all" in requested:
return list(SOURCE_ORDER)
return [source for source in SOURCE_ORDER if source in requested]
@ -196,19 +197,28 @@ def _dedup_key(prop: dict) -> tuple:
def _merge_properties(source_results: dict[str, list[dict]]) -> tuple[list[dict], dict, int]:
merged: dict[str, dict] = {}
seen_keys: set[tuple] = set()
seen_ids: set[str] = set()
counts = {source: 0 for source in SOURCE_ORDER}
deduped = 0
for source in SOURCE_ORDER:
for prop in source_results.get(source, []):
prop_id = prop.get("id")
key = _dedup_key(prop)
if (prop_id is not None and prop_id in merged) or key in seen_keys:
deduped += 1
continue
storage_key = prop_id if prop_id is not None else f"{source}:{len(merged)}"
if prop_id is not None:
prop_id = str(prop_id)
if prop_id in seen_ids:
deduped += 1
continue
seen_ids.add(prop_id)
storage_key = prop_id
else:
key = _dedup_key(prop)
if key in seen_keys:
deduped += 1
continue
seen_keys.add(key)
storage_key = f"{source}:{len(merged)}"
merged[storage_key] = prop
seen_keys.add(key)
counts[source] += 1
return list(merged.values()), counts, deduped
@ -241,13 +251,22 @@ def _store_properties(
if remaining == 0:
return 0
eligible = [prop for prop in props if _property_is_londonish(prop)]
dropped = len(props) - len(eligible)
if dropped:
londonish = [prop for prop in props if _property_is_londonish(prop)]
dropped_outside_area = len(props) - len(londonish)
if dropped_outside_area:
log.debug(
"%s dropped %d properties outside the Greater London-ish postcode filter",
source,
dropped,
dropped_outside_area,
)
eligible = [prop for prop in londonish if matches_strict_buy_listing_filter(prop)]
dropped_non_matching = len(londonish) - len(eligible)
if dropped_non_matching:
log.debug(
"%s dropped %d properties outside the strict buy-listing filters",
source,
dropped_non_matching,
)
selected = eligible if remaining is None else eligible[:remaining]
@ -367,20 +386,16 @@ def _scrape_homecouk(
log.info("home.co.uk cap reached")
return
remaining = _source_remaining(
results, "homecouk", max_properties_per_source
)
if remaining == 0:
log.info("home.co.uk cap reached")
return
for attempt in range(2):
try:
# home.co.uk cannot express the full filter set at source.
# Fetch the outcode page set first; _store_properties applies
# the strict filter and source cap after transformation.
props = homecouk_search_outcode(
client,
outcode,
pc_index,
max_properties=remaining,
max_properties=None,
)
added = _store_properties(
results,
@ -442,19 +457,17 @@ def _scrape_zoopla(
log.info("Zoopla cap reached")
return
remaining = _source_remaining(results, "zoopla", max_properties_per_source)
if remaining == 0:
log.info("Zoopla cap reached")
return
for attempt in range(2):
try:
# Zoopla source-side filters are unverified here. Fetch the
# outcode page set first; _store_properties applies the
# strict filter and source cap after transformation.
props, _ = zoopla_search_outcode(
page,
outcode,
pc_index,
pc_coords,
max_properties=remaining,
max_properties=None,
)
added = _store_properties(
results,
@ -506,9 +519,6 @@ def run_scrape(
output_base = Path(output_dir) if output_dir is not None else DATA_DIR
output_base.mkdir(parents=True, exist_ok=True)
if "zoopla" in selected_sources and pc_coords is None:
pc_coords = build_postcode_coords()
errors: list[str] = []
results = {source: [] for source in SOURCE_ORDER}
started_at = time.time()
@ -539,7 +549,8 @@ def run_scrape(
)
if "zoopla" in selected_sources:
assert pc_coords is not None
if pc_coords is None:
pc_coords = build_postcode_coords()
_scrape_zoopla(
selected_outcodes,
pc_index,
@ -551,19 +562,36 @@ def run_scrape(
merged, source_counts, deduped = _merge_properties(results)
output_path = output_base / "online_listings_buy.parquet"
write_parquet(merged, output_path)
if merged:
write_parquet(merged, output_path)
else:
if output_path.exists():
output_path.unlink()
log.warning("No strict properties to write to %s", output_path)
filtered = [prop for prop in merged if matches_strict_buy_listing_filter(prop)]
filtered_output_path = output_base / "online_listings_buy_filtered.parquet"
if filtered:
write_parquet(filtered, filtered_output_path)
else:
if filtered_output_path.exists():
filtered_output_path.unlink()
log.warning("No strict-filtered properties to write to %s", filtered_output_path)
counts = {
"total": len(merged),
"filtered_total": len(filtered),
"deduped": deduped,
"sources": source_counts,
}
source_summary = " ".join(
f"{source}:{source_counts[source]}" for source in SOURCE_ORDER
)
log.info(
"Sale scrape complete: %d unique (rightmove:%d homecouk:%d zoopla:%d deduped:%d)",
"Sale scrape complete: %d unique, %d strict-filtered (%s deduped:%d)",
len(merged),
source_counts["rightmove"],
source_counts["homecouk"],
source_counts["zoopla"],
len(filtered),
source_summary,
deduped,
)
@ -575,6 +603,7 @@ def run_scrape(
},
"counts": counts,
"path": str(output_path),
"filtered_path": str(filtered_output_path),
"errors": errors,
"elapsed_seconds": round(time.time() - started_at, 3),
}

View file

@ -45,9 +45,10 @@ def write_parquet(properties: list[dict], path: Path) -> None:
remapped = 0
for p in properties:
sub_type = p.get("Property sub-type", "")
if sub_type and sub_type != "Unknown":
current_type = p.get("Property type")
if sub_type and sub_type != "Unknown" and current_type in (None, "", "Other"):
new_type = map_property_type(sub_type)
if new_type != p.get("Property type"):
if new_type != current_type:
p["Property type"] = new_type
remapped += 1
if remapped:

View file

@ -1,4 +1,5 @@
import logging
import math
import re
from constants import MAX_BEDROOMS, PROPERTY_TYPE_MAP, RIGHTMOVE_BASE
@ -29,17 +30,43 @@ def validate_floor_area(sqm: float | None) -> float | None:
return sqm
def parse_int_value(value) -> int | None:
"""Parse an integer-like API value without truncating decimals."""
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
if not math.isfinite(value) or not value.is_integer():
return None
return int(value)
if isinstance(value, str):
cleaned = value.strip().replace(",", "").replace("£", "")
if not re.fullmatch(r"\d+", cleaned):
return None
return int(cleaned)
return None
def parse_display_size(display_size: str | None) -> float | None:
"""Parse displaySize like '499 sq. ft.' or '4,124 sq. ft.' to sqm."""
if not display_size:
return None
# Try sq. ft. first
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*ft", display_size, re.IGNORECASE)
m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))",
display_size,
re.IGNORECASE,
)
if m:
sqft = float(m.group(1).replace(",", ""))
return validate_floor_area(round(sqft * 0.092903, 1))
# Try sq. m.
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*m", display_size, re.IGNORECASE)
m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))",
display_size,
re.IGNORECASE,
)
if m:
return validate_floor_area(round(float(m.group(1).replace(",", "")), 1))
return None
@ -86,7 +113,21 @@ def map_property_type(sub_type: str | None) -> str:
return canonical
# Keyword fallback for compound types not in the map
lower = sub_type.lower()
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower:
excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
):
return "Flats/Maisonettes"
if "semi" in lower and "detach" in lower:
return "Semi-Detached"
@ -158,10 +199,10 @@ def transform_property(
lat, lng = fix_coords(raw_lat, raw_lng)
price_obj = prop.get("price", {})
amount = price_obj.get("amount")
amount = parse_int_value(price_obj.get("amount"))
if not amount:
return None
price = int(amount)
price = amount
if price <= 0:
return None
@ -172,14 +213,23 @@ def transform_property(
# POA / Auction listings have unreliable prices — treat as no price
pq_lower = price_qualifier.lower()
if "poa" in pq_lower or "auction" in pq_lower:
non_comparable_price_terms = (
"poa",
"auction",
"shared ownership",
"shared equity",
"part buy",
"part rent",
"from",
)
if any(term in pq_lower for term in non_comparable_price_terms):
return None
sub_type = prop.get("propertySubType", "")
raw_beds = prop.get("bedrooms", 0) or 0
raw_baths = prop.get("bathrooms", 0) or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0
raw_beds = parse_int_value(prop.get("bedrooms")) or 0
raw_baths = parse_int_value(prop.get("bathrooms")) or 0
bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning(
"Rightmove %s: implausible beds=%d baths=%d (capped to 0)",
@ -197,8 +247,15 @@ def transform_property(
log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng)
return None
property_url = prop.get("propertyUrl") or ""
if not isinstance(property_url, str):
property_url = ""
listing_id = prop.get("id") or property_url
if not listing_id:
return None
return {
"id": prop.get("id"),
"id": listing_id,
"Bedrooms": bedrooms,
"Bathrooms": bathrooms,
"Number of bedrooms & living rooms": bedrooms + bathrooms,
@ -213,7 +270,7 @@ def transform_property(
"price_frequency": "",
"Price qualifier": price_qualifier,
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
"Listing URL": RIGHTMOVE_BASE + prop.get("propertyUrl", ""),
"Listing URL": RIGHTMOVE_BASE + property_url if property_url else "",
"Listing features": key_features,
"first_visible_date": prop.get("firstVisibleDate", ""),
}

View file

@ -24,7 +24,7 @@ import time
from constants import DELAY_BETWEEN_PAGES, MAX_BEDROOMS, PROPERTY_TYPE_MAP, ZOOPLA_BASE
from spatial import PostcodeSpatialIndex
from transform import normalize_sub_type, validate_floor_area
from transform import normalize_sub_type, parse_int_value, validate_floor_area
log = logging.getLogger("zoopla")
@ -106,7 +106,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
const bedsMatch = text.match(/(\d+)\s*beds?/i);
const bathsMatch = text.match(/(\d+)\s*baths?/i);
const recMatch = text.match(/(\d+)\s*reception/i);
const areaMatch = text.match(/([\d,]+)\s*sq\.?\s*ft/i);
const areaSqftMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))/i);
const areaSqmMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))/i);
let tenure = '';
if (/leasehold/i.test(text)) tenure = 'Leasehold';
@ -141,7 +142,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null,
baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null,
receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null,
floor_area_sqft: areaMatch ? parseInt(areaMatch[1].replace(/,/g, '')) : null,
floor_area_sqft: areaSqftMatch ? parseInt(areaSqftMatch[1].replace(/,/g, '')) : null,
floor_area_sqm: areaSqmMatch ? parseFloat(areaSqmMatch[1].replace(/,/g, '')) : null,
address, tenure, property_type,
});
}
@ -181,7 +183,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
const bedsMatch = text.match(/(\d+)\s*beds?/i);
const bathsMatch = text.match(/(\d+)\s*baths?/i);
const recMatch = text.match(/(\d+)\s*reception/i);
const areaMatch = text.match(/([\d,]+)\s*sq\.?\s*ft/i);
const areaSqftMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))/i);
const areaSqmMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))/i);
let address = '';
for (const line of lines) {
@ -225,7 +228,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null,
baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null,
receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null,
floor_area_sqft: areaMatch ? parseInt(areaMatch[1].replace(/,/g, '')) : null,
floor_area_sqft: areaSqftMatch ? parseInt(areaSqftMatch[1].replace(/,/g, '')) : null,
floor_area_sqm: areaSqmMatch ? parseFloat(areaSqmMatch[1].replace(/,/g, '')) : null,
address, tenure, property_type,
});
}
@ -611,7 +615,22 @@ def _map_property_type(raw_type: str | None) -> str:
return canonical
# Keyword fallback
lower = raw_type.lower()
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower or "penthouse" in lower:
excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
or "penthouse" in lower
):
return "Flats/Maisonettes"
if "semi" in lower and "detach" in lower:
return "Semi-Detached"
@ -634,8 +653,8 @@ def transform_property(
Zoopla search cards do not include coordinates, so we resolve lat/lng
from postcodes extracted from the address text."""
price = raw.get("price")
if not price or int(price) <= 0:
price = parse_int_value(raw.get("price"))
if not price or price <= 0:
return None
address = raw.get("address", "")
@ -670,10 +689,10 @@ def transform_property(
if not (49 <= lat <= 56 and -7 <= lng <= 2):
return None
raw_beds = raw.get("beds") or 0
raw_baths = raw.get("baths") or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0
raw_beds = parse_int_value(raw.get("beds")) or 0
raw_baths = parse_int_value(raw.get("baths")) or 0
bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning(
"Zoopla %s: implausible beds=%d baths=%d (capped to 0)",
@ -683,9 +702,13 @@ def transform_property(
# Floor area: convert sq ft to sq m
floor_area_sqm = None
sqft = raw.get("floor_area_sqft")
if sqft:
floor_area_sqm = validate_floor_area(round(sqft * 0.092903, 1))
raw_sqm = raw.get("floor_area_sqm")
if raw_sqm:
floor_area_sqm = validate_floor_area(round(float(raw_sqm), 1))
else:
sqft = raw.get("floor_area_sqft")
if sqft:
floor_area_sqm = validate_floor_area(round(float(sqft) * 0.092903, 1))
listing_id = raw.get("id", "")
listing_url = raw.get("url", "")
@ -704,7 +727,7 @@ def transform_property(
"Leasehold/Freehold": raw.get("tenure") or None,
"Property type": _map_property_type(raw.get("property_type")),
"Property sub-type": normalize_sub_type(raw.get("property_type")),
"price": int(price),
"price": price,
"price_frequency": "",
"Price qualifier": "",
"Total floor area (sqm)": floor_area_sqm,
@ -760,7 +783,18 @@ def search_outcode(
properties = []
dropped = 0
for raw in raw_listings:
transformed = transform_property(raw, pc_index, pc_coords, search_outcode=outcode)
try:
transformed = transform_property(
raw, pc_index, pc_coords, search_outcode=outcode
)
except Exception as exc:
log.warning(
"Zoopla %s property %s failed to transform: %s",
outcode,
raw.get("id", "?"),
exc,
)
transformed = None
if transformed:
properties.append(transformed)
else:

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

View file

@ -1,14 +1,16 @@
import { useMemo, useState } from 'react';
import { useMemo, useState, type MutableRefObject, type ReactNode } from 'react';
import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server';
import type {
FeatureFilters,
FeatureGroup,
FeatureMeta,
FilterExclusion,
HexagonStatsResponse,
} from '../../types';
import { travelFieldKey, type TravelTimeEntry } from '../../hooks/useTravelTime';
import type { HexagonLocation } from '../../lib/external-search';
import { formatStationDistance, type NearbyStation } from '../../lib/nearby-stations';
import {
formatValue,
formatFilterValue,
@ -16,19 +18,22 @@ import {
roundedPercentages,
} from '../../lib/format';
import { groupFeaturesByCategory } from '../../lib/features';
import { getPoiCategoryLogoUrl } from '../../lib/map-utils';
import {
PARTY_FEATURE_COLORS,
STACKED_GROUPS,
STACKED_ENUM_GROUPS,
STACKED_SEGMENT_COLORS,
} from '../../lib/consts';
import { useNearbyStations } from '../../hooks/useNearbyStations';
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
import { DualHistogram, LoadingSkeleton } from './DualHistogram';
import EnumBarChart from './EnumBarChart';
import StackedBarChart from './StackedBarChart';
import StackedEnumChart from './StackedEnumChart';
import PriceHistoryChart from './PriceHistoryChart';
import ExternalSearchLinks from './ExternalSearchLinks';
import { InfoIcon } from '../ui/icons';
import { InfoIcon, TransitIcon } from '../ui/icons';
import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader';
import { FeatureInfoPopup } from '../ui/FeatureInfoPopup';
import { EmptyState } from '../ui/EmptyState';
@ -54,6 +59,9 @@ interface AreaPaneProps {
shareCode?: string;
isGroupExpanded: (name: string) => boolean;
onToggleGroup: (name: string) => void;
scrollTopRef?: MutableRefObject<number>;
scrollRestoreKey?: string | null;
scrollSaveDisabled?: boolean;
}
function normalizePercentageSegments<T extends { value: number }>(segments: T[]): T[] {
@ -75,6 +83,136 @@ function filterValueFormat(feature?: FeatureMeta) {
};
}
const STATION_GROUP_NAME = 'Transport';
const STATION_GROUP_NAMES = new Set([STATION_GROUP_NAME, 'Public Transport']);
function MetricTextLabel({ children }: { children: ReactNode }) {
return (
<span className="block truncate text-[13px] font-medium leading-5 text-warm-900 dark:text-warm-100">
{children}
</span>
);
}
function MetricFeatureLabel({
feature,
onShowInfo,
label,
aboutLabel,
}: {
feature: FeatureMeta;
onShowInfo: (feature: FeatureMeta) => void;
label?: string;
aboutLabel: string;
}) {
return (
<div className="flex min-w-0 items-center gap-1.5">
<MetricTextLabel>{label ?? ts(feature.name)}</MetricTextLabel>
{feature.detail && (
<button
type="button"
onClick={() => onShowInfo(feature)}
className="-m-1 shrink-0 rounded p-1 text-warm-400 hover:bg-warm-100 hover:text-warm-700 dark:hover:bg-navy-800 dark:hover:text-warm-200"
title={aboutLabel}
aria-label={aboutLabel}
>
<InfoIcon className="h-3.5 w-3.5" />
</button>
)}
</div>
);
}
function MetricRow({
label,
chart,
value,
valueTitle,
className = '',
}: {
label: ReactNode;
chart?: ReactNode;
value?: ReactNode;
valueTitle?: string;
className?: string;
}) {
return (
<div
className={`grid min-h-10 grid-cols-[minmax(0,1fr)_6.5rem_minmax(3.5rem,auto)] items-center gap-3 py-1.5 ${className}`}
>
<div className="min-w-0">{label}</div>
<div className="w-[6.5rem] justify-self-end">{chart}</div>
<div
className="min-w-[3.5rem] max-w-[7rem] truncate text-right text-sm font-semibold leading-tight tabular-nums text-navy-950 dark:text-warm-50"
title={valueTitle}
>
{value}
</div>
</div>
);
}
function NearbyStationsCard({ location }: { location: HexagonLocation }) {
const { t } = useTranslation();
const origin = useMemo(
() => ({ lat: location.lat, lon: location.lon }),
[location.lat, location.lon]
);
const { stations, loading } = useNearbyStations(origin);
return (
<div className="py-1.5">
<div className="flex items-center gap-2 py-1">
<TransitIcon className="h-4 w-4 text-teal-600 dark:text-teal-400" />
<MetricTextLabel>{t('areaPane.closestStations')}</MetricTextLabel>
{loading && (
<span className="ml-auto h-3 w-3 rounded-full border-2 border-teal-600 border-t-transparent dark:border-teal-400 dark:border-t-transparent animate-spin" />
)}
</div>
{stations.length > 0 ? (
<ol className="divide-y divide-warm-100 dark:divide-navy-800">
{stations.map((station) => (
<NearbyStationRow key={station.id} station={station} />
))}
</ol>
) : (
<div className="py-2 text-sm text-warm-500 dark:text-warm-400">
{loading ? t('common.loading') : t('areaPane.noNearbyStations')}
</div>
)}
</div>
);
}
function NearbyStationRow({ station }: { station: NearbyStation }) {
const icon = getPoiCategoryLogoUrl(station.category, station.icon_category);
return (
<li className="flex items-center gap-2 px-3 py-2">
{icon ? (
<img
src={icon}
alt=""
aria-hidden="true"
loading="lazy"
className="h-5 w-5 shrink-0 rounded-[3px] bg-white object-contain p-0.5"
/>
) : (
<TransitIcon className="h-5 w-5 shrink-0 text-warm-400 dark:text-warm-500" />
)}
<div className="min-w-0 flex-1">
<div className="truncate text-sm font-medium text-warm-900 dark:text-warm-100">
{station.name}
</div>
<div className="text-xs text-warm-500 dark:text-warm-400">{ts(station.category)}</div>
</div>
<span className="shrink-0 text-sm font-semibold tabular-nums text-teal-700 dark:text-teal-400">
{formatStationDistance(station.distanceKm)}
</span>
</li>
);
}
export default function AreaPane({
stats,
globalFeatures,
@ -91,6 +229,9 @@ export default function AreaPane({
shareCode,
isGroupExpanded,
onToggleGroup,
scrollTopRef,
scrollRestoreKey,
scrollSaveDisabled,
}: AreaPaneProps) {
const { t } = useTranslation();
const propertyCount = stats?.count;
@ -99,7 +240,19 @@ export default function AreaPane({
const filteredStatsEmpty = filtersActive && statsUseFilters && stats?.count === 0;
const showFlipToggleCallout = filteredStatsEmpty && unfilteredCount !== 0;
const featureGroups = useMemo(() => groupFeaturesByCategory(globalFeatures), [globalFeatures]);
const displayFeatureGroups = useMemo<FeatureGroup[]>(() => {
if (!hexagonLocation || featureGroups.some((group) => STATION_GROUP_NAMES.has(group.name))) {
return featureGroups;
}
return [{ name: STATION_GROUP_NAME, features: [] }, ...featureGroups];
}, [featureGroups, hexagonLocation]);
const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null);
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey: scrollRestoreKey ?? hexagonId,
scrollTopRef,
suspendSave: scrollSaveDisabled ?? (loading && stats == null),
});
const numericByName = useMemo(() => {
if (!stats) return new Map();
@ -164,7 +317,7 @@ export default function AreaPane({
<>
<div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && stats != null} />
<div className="flex-1 overflow-y-auto">
<div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto">
<div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950">
<div className="space-y-3 p-3">
<div className="flex items-start justify-between gap-3">
@ -300,20 +453,22 @@ export default function AreaPane({
{stats.price_history &&
(() => {
const uniqueYears = new Set(stats.price_history.map((p) => Math.floor(p.year)));
return uniqueYears.size > 1;
})() && (
<div className="mx-3 mt-2 bg-warm-50 dark:bg-warm-800 rounded p-2">
<span className="text-xs text-warm-700 dark:text-warm-300">
{t('areaPane.priceHistory')}
</span>
<PriceHistoryChart points={stats.price_history} />
</div>
)}
{featureGroups.map((group) => {
return uniqueYears.size > 1 ? (
<div className="mx-3 mt-2 bg-warm-50 dark:bg-warm-800 rounded p-2">
<span className="text-xs text-warm-700 dark:text-warm-300">
{t('areaPane.priceHistory')}
</span>
<PriceHistoryChart points={stats.price_history} />
</div>
) : null;
})()}
{displayFeatureGroups.map((group) => {
const showNearbyStations =
hexagonLocation != null && STATION_GROUP_NAMES.has(group.name);
const hasData = group.features.some(
(feature) => numericByName.has(feature.name) || enumByName.has(feature.name)
);
if (!hasData) return null;
if (!hasData && !showNearbyStations) return null;
const stackedCharts = STACKED_GROUPS[group.name];
const stackedEnumCharts = STACKED_ENUM_GROUPS[group.name];
@ -332,10 +487,11 @@ export default function AreaPane({
name={group.name}
expanded={expanded}
onToggle={() => onToggleGroup(group.name)}
className="px-3 py-2.5 text-sm font-bold text-warm-500 bg-warm-50 dark:bg-warm-900 dark:text-warm-400 sticky top-0 z-10 hover:bg-warm-100 dark:hover:bg-warm-800"
className="area-pane-group-header sticky top-0 z-10 bg-white px-3 pb-1.5 pt-4 text-[11px] font-bold uppercase tracking-wide text-warm-500 hover:bg-warm-50 dark:bg-navy-950 dark:text-warm-400 dark:hover:bg-navy-900"
/>
{expanded && (
<div className="px-3 py-2 space-y-3">
<div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800">
{showNearbyStations && <NearbyStationsCard location={hexagonLocation} />}
{stackedCharts?.map((chart) => {
const segments = chart.components
.map((name) => ({
@ -445,21 +601,17 @@ export default function AreaPane({
: undefined;
return (
<div
<MetricRow
key={feature.name}
className="bg-warm-50 dark:bg-warm-800 rounded p-2"
>
<div className="flex justify-between items-baseline">
<FeatureLabel
label={
<MetricFeatureLabel
feature={feature}
onShowInfo={setInfoFeature}
className="mr-2"
aboutLabel={t('filters.aboutData')}
/>
<span className="text-xs font-semibold text-teal-700 dark:text-teal-400 whitespace-nowrap">
{formatValue(numericStats.mean, feature)}
</span>
</div>
{numericStats.histogram &&
}
chart={
numericStats.histogram &&
(globalHistogram ? (
<DualHistogram
localCounts={numericStats.histogram.counts}
@ -476,6 +628,8 @@ export default function AreaPane({
: feature.raw
)
}
integerAxisLabels={feature.step === 1}
compact
/>
) : (
<DualHistogram
@ -491,9 +645,18 @@ export default function AreaPane({
: feature.raw
)
}
integerAxisLabels={feature.step === 1}
compact
/>
))}
</div>
))
}
value={formatValue(numericStats.mean, feature)}
valueTitle={
globalMean != null
? `${t('areaPane.nationalAvg')}: ${formatValue(globalMean)}`
: undefined
}
/>
);
}

View file

@ -0,0 +1,26 @@
import { describe, expect, it } from 'vitest';
import { compactHistogramLabel } from './DualHistogram';
describe('compactHistogramLabel', () => {
it('rounds low-cardinality count labels to integers', () => {
const fmt = (value: number) => value.toFixed(2);
const labels = [0, 0.99, 2.98, 4.96, 5.95].map((center, index) =>
compactHistogramLabel(index, 5, 0, 5.95, center, fmt, true)
);
expect(labels).toEqual(['0', '1', '3', '5', '6+']);
});
it('labels the first integer count bucket as zero when it means below one', () => {
const fmt = (value: number) => value.toFixed(2);
expect(compactHistogramLabel(0, 5, 0.99, 5.95, 0.99, fmt, true)).toBe('0');
});
it('keeps fractional labels when integer labels are not requested', () => {
const fmt = (value: number) => value.toFixed(2);
expect(compactHistogramLabel(1, 5, 0, 5.95, 0.99, fmt, false)).toBe('0.99');
});
});

View file

@ -30,6 +30,42 @@ function pickTicks(min: number, max: number, count: number): number[] {
return ticks;
}
function isLowCardinalityHistogram(counts: number[], p1: number, p99: number): boolean {
return counts.length > 0 && counts.length <= 10 && p99 > p1 && p99 - p1 <= 10;
}
export function compactHistogramLabel(
index: number,
barCount: number,
p1: number,
p99: number,
center: number,
formatLabel: (value: number) => string,
integerLabels = false
): string {
const formatAxisValue = (value: number) =>
integerLabels ? Math.round(value).toLocaleString() : formatLabel(value);
if (barCount <= 1) return formatAxisValue(center);
const middleBins = barCount - 2;
if (index === 0) {
if (!integerLabels) return `<${formatLabel(p1)}`;
const firstBoundary = Math.ceil(p1);
return firstBoundary <= 1 ? '0' : `<${firstBoundary.toLocaleString()}`;
}
if (index === barCount - 1) {
if (!integerLabels) return `${formatLabel(p99)}+`;
return `${Math.ceil(p99).toLocaleString()}+`;
}
const middleWidth = middleBins > 0 ? (p99 - p1) / middleBins : 0;
if (Math.abs(middleWidth - 1) < 0.001) {
return formatAxisValue(p1 + index - 1);
}
return formatAxisValue(center);
}
export function DualHistogram({
localCounts,
globalCounts,
@ -38,6 +74,8 @@ export function DualHistogram({
globalMean,
meanLabel,
formatLabel,
compact = false,
integerAxisLabels = false,
}: {
localCounts: number[];
globalCounts: number[];
@ -46,9 +84,15 @@ export function DualHistogram({
globalMean?: number;
meanLabel?: string;
formatLabel?: (value: number) => string;
compact?: boolean;
integerAxisLabels?: boolean;
}) {
const { t } = useTranslation();
const targetBars = 25;
const showCompactAxisLabels =
compact &&
isLowCardinalityHistogram(localCounts, p1, p99) &&
isLowCardinalityHistogram(globalCounts, p1, p99);
const targetBars = compact ? (showCompactAxisLabels ? localCounts.length : 16) : 25;
const localBars = downsampleBars(localCounts, targetBars);
const globalBars = downsampleBars(globalCounts, targetBars);
@ -59,6 +103,8 @@ export function DualHistogram({
const fmt =
formatLabel ?? ((v: number) => (Number.isInteger(v) ? v.toLocaleString() : v.toFixed(1)));
if (barCount === 0) return null;
// Compute center value for each bar.
// Bar 0 = low outlier, bars 1..n-2 = middle (p1 to p99), bar n-1 = high outlier.
const middleBins = Math.max(barCount - 2, 0);
@ -97,6 +143,60 @@ export function DualHistogram({
? { right: 0 }
: { left: '50%', transform: 'translateX(-50%)' };
if (compact) {
const axisLabels = showCompactAxisLabels
? barCenters.map((center, index) =>
compactHistogramLabel(index, barCount, p1, p99, center, fmt, integerAxisLabels)
)
: [];
const chartTitle = [
`${fmt(p1)} - ${fmt(p99)}`,
globalMean != null ? `${meanLabel ?? t('areaPane.nationalAvg')}: ${fmt(globalMean)}` : null,
]
.filter(Boolean)
.join('\n');
return (
<div className={showCompactAxisLabels ? 'h-10' : 'h-7'} title={chartTitle}>
<div
className={`${showCompactAxisLabels ? 'h-7' : 'h-full'} relative flex items-end gap-[2px]`}
>
{Array.from({ length: barCount }).map((_, index) => {
const globalHeight = (globalBars[index] / globalMax) * 100;
const localHeight = (localBars[index] / localMax) * 100;
return (
<div key={index} className="relative flex h-full min-w-[2px] flex-1 items-end">
<div
className="absolute bottom-0 left-0 right-0 rounded-t-[2px] bg-warm-300/45 dark:bg-warm-600/50"
style={{ height: `${Math.max(globalHeight, globalBars[index] > 0 ? 8 : 0)}%` }}
/>
{localBars[index] > 0 && (
<div
className="absolute bottom-0 left-[22%] right-[22%] rounded-t-[2px] bg-teal-600 dark:bg-teal-400"
style={{ height: `${Math.max(localHeight, 12)}%` }}
/>
)}
</div>
);
})}
</div>
{showCompactAxisLabels && (
<div className="mt-0.5 flex gap-[2px]">
{axisLabels.map((label, index) => (
<span
key={`${label}-${index}`}
className="min-w-[2px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={label}
>
{label}
</span>
))}
</div>
)}
</div>
);
}
return (
<div className="mt-1">
<div className={showMeanMarker ? 'relative pt-5' : 'relative'}>
@ -152,35 +252,29 @@ export function DualHistogram({
function SkeletonHistogram() {
return (
<div className="bg-warm-50 dark:bg-warm-800 rounded p-2 animate-pulse">
<div className="flex justify-between items-baseline">
<div className="h-3 w-24 bg-warm-200 dark:bg-warm-700 rounded" />
<div className="h-3 w-10 bg-warm-200 dark:bg-warm-700 rounded" />
</div>
<div className="flex items-end gap-px h-10 mt-2">
{Array.from({ length: 15 }).map((_, i) => (
<div className="grid min-h-10 animate-pulse grid-cols-[minmax(0,1fr)_6.5rem_minmax(3.5rem,auto)] items-center gap-3 py-1.5">
<div className="h-3 w-24 rounded bg-warm-200 dark:bg-warm-700" />
<div className="flex h-7 items-end gap-[2px]">
{Array.from({ length: 12 }).map((_, i) => (
<div
key={i}
className="flex-1 bg-warm-200 dark:bg-warm-700 rounded-t-sm min-w-[2px]"
style={{ height: `${20 + Math.sin(i * 0.7) * 30 + 30}%` }}
className="min-w-[2px] flex-1 rounded-t-[2px] bg-warm-200 dark:bg-warm-700"
style={{ height: `${22 + Math.sin(i * 0.7) * 28 + 30}%` }}
/>
))}
</div>
<div className="flex justify-between mt-1">
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
</div>
<div className="h-3 w-10 justify-self-end rounded bg-warm-200 dark:bg-warm-700" />
</div>
);
}
export function LoadingSkeleton() {
return (
<div className="p-3 space-y-4">
<div className="space-y-4 p-3">
{[0, 1, 2].map((groupIdx) => (
<div key={groupIdx}>
<div className="h-3 w-20 bg-warm-200 dark:bg-warm-700 rounded animate-pulse mb-2" />
<div className="space-y-3">
<div className="mb-2 h-3 w-20 animate-pulse rounded bg-warm-200 dark:bg-warm-700" />
<div className="divide-y divide-warm-100 dark:divide-navy-800">
{Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => (
<SkeletonHistogram key={i} />
))}

View file

@ -1,16 +1,34 @@
import { ts } from '../../i18n/server';
import { getEnumValueColor } from '../../lib/consts';
function shortenAxisLabel(label: string, total: number): string {
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label)
.slice(0, total <= 5 ? 3 : 2)
.join('');
}
export default function EnumBarChart({
counts,
globalCounts,
featureName,
compact = false,
}: {
counts: Record<string, number>;
globalCounts?: Record<string, number>;
featureName: string;
compact?: boolean;
}) {
const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA);
if (entries.length === 0) return null;
const localTotal = entries.reduce((sum, [, c]) => sum + c, 0);
// When global counts are available, normalize both to percentages for comparison
@ -28,6 +46,71 @@ export default function EnumBarChart({
// Fallback to raw count scaling when no global data
const maxCount = Math.max(...entries.map(([, count]) => count), 1);
if (compact) {
const title = entries
.map(([label, count]) => {
const localPct = localTotal > 0 ? (count / localTotal) * 100 : 0;
const globalPct =
hasGlobal && globalTotal > 0 ? ((globalCounts[label] ?? 0) / globalTotal) * 100 : null;
return `${ts(label)}: ${count.toLocaleString()} (${localPct.toFixed(1)}%)${
globalPct != null ? ` / ${globalPct.toFixed(1)}%` : ''
}`;
})
.join('\n');
return (
<div className="h-10" title={title}>
<div className="flex h-7 items-end gap-[2px]">
{entries.map(([label, count]) => {
const localPct = localTotal > 0 ? count / localTotal : 0;
const globalPct = hasGlobal ? (globalCounts[label] ?? 0) / globalTotal : 0;
const localHeight = hasGlobal
? maxPct > 0
? (localPct / maxPct) * 100
: 0
: (count / maxCount) * 100;
const globalHeight = hasGlobal && maxPct > 0 ? (globalPct / maxPct) * 100 : 0;
const color = getEnumValueColor(featureName, label);
return (
<div key={label} className="relative flex h-full min-w-[3px] flex-1 items-end">
{hasGlobal && (
<div
className="absolute bottom-0 left-0 right-0 rounded-t-[2px] bg-warm-300/45 dark:bg-warm-600/50"
style={{ height: `${Math.max(globalHeight, globalPct > 0 ? 8 : 0)}%` }}
/>
)}
{count > 0 && (
<div
className="absolute bottom-0 left-[18%] right-[18%] rounded-t-[2px]"
style={{
height: `${Math.max(localHeight, 12)}%`,
backgroundColor: `rgb(${color[0]},${color[1]},${color[2]})`,
}}
/>
)}
</div>
);
})}
</div>
<div className="mt-0.5 flex gap-[2px]">
{entries.map(([label]) => {
const translated = ts(label);
return (
<span
key={label}
className="min-w-[3px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={translated}
>
{shortenAxisLabel(translated, entries.length)}
</span>
);
})}
</div>
</div>
);
}
return (
<div className="space-y-1 mt-1">
{entries.map(([label, count]) => {

View file

@ -3,35 +3,18 @@ import { useTranslation } from 'react-i18next';
export default function HistogramLegend() {
const { t } = useTranslation();
return (
<div className="mx-3 mt-3 bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5 text-xs">
<div className="space-y-1.5">
<div className="flex items-center gap-2">
<div className="w-3 h-3 bg-teal-500 dark:bg-teal-400 rounded" />
<span className="text-warm-700 dark:text-warm-300">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.tealBars')}
</span>{' '}
{t('histogramLegend.tealBarsDesc')}
</span>
</div>
<div className="flex items-center gap-2">
<div className="w-3 h-3 bg-warm-300/60 dark:bg-warm-600/60 rounded" />
<span className="text-warm-700 dark:text-warm-300">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.greyBars')}
</span>{' '}
{t('histogramLegend.greyBarsDesc')}
</span>
</div>
<div className="flex items-center gap-2">
<div className="w-3 h-px border-t border-dashed border-warm-500 dark:border-warm-400" />
<span className="text-warm-700 dark:text-warm-300">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.dashedLine')}
</span>{' '}
{t('histogramLegend.dashedLineDesc')}
</span>
</div>
<div className="mx-3 mt-3 flex flex-wrap items-center gap-x-3 gap-y-1 rounded border border-warm-200 bg-white px-2.5 py-1.5 text-[10px] text-warm-500 dark:border-navy-800 dark:bg-navy-950/60 dark:text-warm-400">
<div className="flex items-center gap-1.5">
<div className="h-2.5 w-2 rounded-[2px] bg-teal-600 dark:bg-teal-400" />
<span className="font-medium text-warm-700 dark:text-warm-200">
{t('histogramLegend.tealBars')}
</span>
</div>
<div className="flex items-center gap-1.5">
<div className="h-2.5 w-2 rounded-[2px] bg-warm-300/70 dark:bg-warm-600/70" />
<span className="font-medium text-warm-700 dark:text-warm-200">
{t('histogramLegend.greyBars')}
</span>
</div>
</div>
);

View file

@ -147,6 +147,8 @@ export default function MapPage({
const pendingCurrentLocationFlyToRef = useRef<{ lat: number; lng: number } | null>(null);
const pendingLocationSearchFlyToRef = useRef<PendingFlyTo | null>(null);
const mobileDrawerPanelRectRef = useRef<DOMRectReadOnly | null>(null);
const areaPaneScrollTopRef = useRef(0);
const propertiesPaneScrollTopRef = useRef(0);
const getMobileMapFlyToOptions = useCallback((): MapFlyToOptions | undefined => {
if (!isMobile) return undefined;
@ -558,6 +560,11 @@ export default function MapPage({
shareCode={shareCode}
isGroupExpanded={isAreaGroupExpanded}
onToggleGroup={toggleAreaGroup}
scrollTopRef={areaPaneScrollTopRef}
scrollRestoreKey={
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
}
scrollSaveDisabled={loadingAreaStats && areaStats == null}
/>
</Suspense>
);
@ -570,6 +577,11 @@ export default function MapPage({
loading={loadingProperties}
hexagonId={selectedHexagon?.id || null}
onLoadMore={handleLoadMoreProperties}
scrollTopRef={propertiesPaneScrollTopRef}
scrollRestoreKey={
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
}
scrollSaveDisabled={loadingProperties && properties.length === 0}
/>
</Suspense>
);

View file

@ -3,7 +3,7 @@ import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server';
import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups';
import { trackEvent } from '../../lib/analytics';
import { POI_CATEGORY_LOGOS } from '../../lib/consts';
import { getPoiCategoryLogoUrl } from '../../lib/map-utils';
import type { POICategoryGroup } from '../../types';
import InfoPopup from '../ui/InfoPopup';
import { SearchInput } from '../ui/SearchInput';
@ -188,7 +188,7 @@ export default function POIPane({
<div className="px-3 py-2">
<PillGroup>
{group.categories.map((category) => {
const logo = POI_CATEGORY_LOGOS[category];
const logo = getPoiCategoryLogoUrl(category);
return (
<PillToggle
key={category}

View file

@ -1,8 +1,9 @@
import { useMemo, useState, useEffect } from 'react';
import { useMemo, useState, useEffect, type MutableRefObject } from 'react';
import { useTranslation } from 'react-i18next';
import { Property } from '../../types';
import { formatDuration, formatAge, formatNumber, formatTransactionDate } from '../../lib/format';
import { getNum } from '../../lib/property-fields';
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
import InfoPopup from '../ui/InfoPopup';
import { SearchInput } from '../ui/SearchInput';
import { EmptyState } from '../ui/EmptyState';
@ -17,6 +18,9 @@ interface PropertiesPaneProps {
hexagonId: string | null;
onLoadMore: () => void;
onNavigateToSource?: (slug: string) => void;
scrollTopRef?: MutableRefObject<number>;
scrollRestoreKey?: string | null;
scrollSaveDisabled?: boolean;
}
export function PropertiesPane({
@ -26,10 +30,18 @@ export function PropertiesPane({
hexagonId,
onLoadMore,
onNavigateToSource,
scrollTopRef,
scrollRestoreKey,
scrollSaveDisabled,
}: PropertiesPaneProps) {
const { t } = useTranslation();
const [search, setSearch] = useState('');
const [showInfo, setShowInfo] = useState(false);
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey: scrollRestoreKey ?? hexagonId,
scrollTopRef,
suspendSave: scrollSaveDisabled ?? (loading && properties.length === 0),
});
useEffect(() => {
setSearch('');
@ -60,7 +72,7 @@ export function PropertiesPane({
return (
<div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && properties.length > 0} />
<div className="flex-1 overflow-y-auto">
<div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto">
{showInfo && (
<InfoPopup
title={t('propertyCard.propertyData')}

View file

@ -12,6 +12,7 @@ interface StackedBarChartProps {
segments: Segment[];
total: number;
colorMap: Record<string, string>;
compact?: boolean;
}
/** Strip common suffixes/prefixes to produce short legend labels */
@ -28,7 +29,27 @@ function shortenLabel(name: string): string {
.trim();
}
export default function StackedBarChart({ segments, total, colorMap }: StackedBarChartProps) {
function shortenAxisLabel(name: string, total: number): string {
const label = shortenLabel(name);
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label)
.slice(0, total <= 5 ? 3 : 2)
.join('');
}
export default function StackedBarChart({
segments,
total,
colorMap,
compact = false,
}: StackedBarChartProps) {
const { t } = useTranslation();
const sortedSegments = useMemo(() => [...segments].sort((a, b) => b.value - a.value), [segments]);
const roundedPcts = useMemo(
@ -55,6 +76,53 @@ export default function StackedBarChart({ segments, total, colorMap }: StackedBa
return color;
};
if (compact) {
const maxValue = Math.max(...sortedSegments.map((segment) => segment.value), 1);
const showAxisLabels = sortedSegments.length <= 8;
const title = sortedSegments
.map((segment, i) => {
const label = shortenLabel(ts(segment.name));
return `${label}: ${formatValue(segment.value)} (${roundedPcts[i].toFixed(1)}%)`;
})
.join('\n');
return (
<div className={showAxisLabels ? 'h-10' : 'h-7'} title={title}>
<div className={`${showAxisLabels ? 'h-7' : 'h-full'} flex items-end gap-[2px]`}>
{sortedSegments.map((segment) => {
const height = (segment.value / maxValue) * 100;
return (
<div
key={segment.name}
className="min-w-[3px] flex-1 rounded-t-[2px]"
style={{
height: `${Math.max(height, 12)}%`,
backgroundColor: colorFor(segment.name),
}}
/>
);
})}
</div>
{showAxisLabels && (
<div className="mt-0.5 flex gap-[2px]">
{sortedSegments.map((segment) => {
const label = shortenLabel(ts(segment.name));
return (
<span
key={segment.name}
className="min-w-[3px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={label}
>
{shortenAxisLabel(label, sortedSegments.length)}
</span>
);
})}
</div>
)}
</div>
);
}
return (
<div className="space-y-1.5">
{/* Stacked bar */}

View file

@ -7,6 +7,7 @@ interface StackedEnumChartProps {
components: { label: string; stats: EnumFeatureStats }[];
valueOrder: string[];
valueColors: string[];
compact?: boolean;
}
/** Strip common suffixes to produce short row labels */
@ -14,10 +15,24 @@ function shortenLabel(name: string): string {
return name.replace(/ risk$/, '');
}
function shortenAxisLabel(name: string): string {
const label = shortenLabel(name);
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label).slice(0, 3).join('');
}
export default function StackedEnumChart({
components,
valueOrder,
valueColors,
compact = false,
}: StackedEnumChartProps) {
const { t } = useTranslation();
const visibleRows = components.filter(({ stats }) => {
@ -35,6 +50,63 @@ export default function StackedEnumChart({
);
}
if (compact) {
return (
<div className="divide-y divide-warm-100 dark:divide-navy-800">
{visibleRows.map(({ label, stats }) => {
const counts = valueOrder.map((value) => stats.counts[value] ?? 0);
const total = counts.reduce((a, b) => a + b, 0);
const roundedPcts = roundedPercentages(counts, total, 0);
const title = valueOrder
.map((value, i) => `${ts(value)}: ${counts[i]} (${roundedPcts[i]}%)`)
.join('\n');
return (
<div
key={label}
className="grid min-h-8 grid-cols-[minmax(0,1fr)_6.5rem] items-center gap-3 py-1.5"
>
<span className="truncate text-xs font-medium text-warm-800 dark:text-warm-200">
{shortenLabel(ts(label))}
</span>
<div
className="flex h-5 overflow-hidden rounded-sm bg-warm-200 dark:bg-warm-700"
title={title}
>
{valueOrder.map((value, i) => {
const count = counts[i];
const pct = (count / total) * 100;
if (pct < 0.5) return null;
return (
<div
key={value}
className="h-full"
style={{
width: `${pct}%`,
backgroundColor: valueColors[i],
}}
/>
);
})}
</div>
</div>
);
})}
<div className="ml-auto grid w-[6.5rem] grid-flow-col auto-cols-fr gap-1 pt-1">
{valueOrder.map((value) => (
<span
key={value}
className="truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={ts(value)}
>
{shortenAxisLabel(ts(value))}
</span>
))}
</div>
</div>
);
}
return (
<div className="space-y-1.5">
{visibleRows.map(({ label, stats }) => {

View file

@ -110,7 +110,7 @@ export function ActiveFiltersPanel({
>
<button
onClick={onToggleCollapsed}
className="shrink-0 flex items-center justify-between border-y border-l-4 border-teal-300 border-l-teal-600 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:border-l-teal-300 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
className="shrink-0 flex items-center justify-between border-y border-teal-300 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
>
<div className="flex items-center gap-2">
<span className="text-sm font-bold text-navy-950 dark:text-warm-100">

View file

@ -110,7 +110,7 @@ export function AddFilterPanel({
>
<button
onClick={onToggleCollapsed}
className="shrink-0 flex items-center justify-between border-y border-l-4 border-teal-300 border-l-teal-600 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:border-l-teal-300 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
className="shrink-0 flex items-center justify-between border-y border-teal-300 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
>
<span className="text-sm font-bold text-navy-950 dark:text-warm-100">
{t('filters.addFilter')}
@ -122,8 +122,8 @@ export function AddFilterPanel({
</button>
{(!collapsed || !isLicensed) && (
<div className="flex min-h-0 flex-1 flex-col">
<div className="flex min-h-0 flex-1 flex-col overflow-y-auto">
{!collapsed && (
{!collapsed && (
<div className="min-h-0 flex-1 overflow-y-auto">
<FeatureBrowser
availableFeatures={availableFeatures}
allFeatures={allFeatures}
@ -136,37 +136,37 @@ export function AddFilterPanel({
travelTimeEntries={travelTimeEntries}
onAddTravelTimeEntry={onAddTravelTimeEntry}
/>
)}
{!isLicensed && (
<div className="mt-auto shrink-0 flex flex-col items-center px-5 pt-4 pb-0 border-t border-warm-200 dark:border-warm-700">
<p className="text-sm text-warm-600 dark:text-warm-400 text-center leading-relaxed mb-1">
{t('filters.upgradePrompt')}
</p>
<p className="text-xs text-warm-400 dark:text-warm-500 text-center mb-4">
{t('filters.oneTimeLifetime')}
</p>
<button
onClick={onUpgradeClick}
className="px-5 py-2.5 rounded-lg bg-gradient-to-r from-teal-500 to-teal-600 hover:from-teal-600 hover:to-teal-700 text-white font-medium text-sm shadow-sm hover:shadow-md"
>
{t('filters.upgradeToFullMap')}
</button>
<svg
viewBox="0 120 1600 230"
className="w-full mt-4 block shrink-0"
preserveAspectRatio="xMidYMax meet"
>
<path
d="M0,350 C400,150 1200,150 1600,350 Z"
className="fill-green-500 dark:fill-green-600"
/>
<path d="M100,350 C450,180 1150,180 1500,350 Z" fill="#000" opacity="0.08" />
<path d="M250,350 C550,220 1050,220 1350,350 Z" fill="#000" opacity="0.06" />
<image href="/house.png" x="735" y="110" width="130" height="120" />
</svg>
</div>
)}
</div>
</div>
)}
{!isLicensed && (
<div className="mt-auto shrink-0 flex flex-col items-center px-5 pt-4 pb-0 border-t border-warm-200 dark:border-warm-700">
<p className="text-sm text-warm-600 dark:text-warm-400 text-center leading-relaxed mb-1">
{t('filters.upgradePrompt')}
</p>
<p className="text-xs text-warm-400 dark:text-warm-500 text-center mb-4">
{t('filters.oneTimeLifetime')}
</p>
<button
onClick={onUpgradeClick}
className="px-5 py-2.5 rounded-lg bg-gradient-to-r from-teal-500 to-teal-600 hover:from-teal-600 hover:to-teal-700 text-white font-medium text-sm shadow-sm hover:shadow-md"
>
{t('filters.upgradeToFullMap')}
</button>
<svg
viewBox="0 120 1600 230"
className="w-full mt-4 block shrink-0"
preserveAspectRatio="xMidYMax meet"
>
<path
d="M0,350 C400,150 1200,150 1600,350 Z"
className="fill-green-500 dark:fill-green-600"
/>
<path d="M100,350 C450,180 1150,180 1500,350 Z" fill="#000" opacity="0.08" />
<path d="M250,350 C550,220 1050,220 1350,350 Z" fill="#000" opacity="0.06" />
<image href="/house.png" x="735" y="110" width="130" height="120" />
</svg>
</div>
)}
</div>
)}
</div>

View file

@ -1,5 +1,7 @@
import { useRef, useCallback, useEffect, useId, type ReactNode } from 'react';
import { useCallback, useEffect, useId, type ReactNode } from 'react';
import { createPortal } from 'react-dom';
import { useClickOutside } from '../../hooks/useClickOutside';
import { useModalA11y } from '../../hooks/useModalA11y';
import { CloseIcon } from './icons';
import { IconButton } from './IconButton';
@ -11,8 +13,7 @@ interface InfoPopupProps {
}
export default function InfoPopup({ title, children, onClose, sourceLink }: InfoPopupProps) {
const popupRef = useRef<HTMLDivElement>(null);
const previouslyFocusedRef = useRef<HTMLElement | null>(null);
const popupRef = useModalA11y();
const titleId = useId();
const handleClose = useCallback(() => {
@ -29,20 +30,9 @@ export default function InfoPopup({ title, children, onClose, sourceLink }: Info
return () => document.removeEventListener('keydown', handleKeyDown);
}, [onClose]);
useEffect(() => {
previouslyFocusedRef.current = document.activeElement as HTMLElement | null;
const firstFocusable = popupRef.current?.querySelector<HTMLElement>(
'button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])'
);
(firstFocusable ?? popupRef.current)?.focus();
return () => {
previouslyFocusedRef.current?.focus?.();
};
}, []);
return (
const popup = (
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/30 p-4"
className="fixed inset-0 z-[10000] flex items-center justify-center bg-black/50 p-4 dark:bg-black/70"
role="presentation"
>
<div
@ -73,4 +63,8 @@ export default function InfoPopup({ title, children, onClose, sourceLink }: Info
</div>
</div>
);
if (typeof document === 'undefined') return popup;
return createPortal(popup, document.body);
}

View file

@ -160,7 +160,7 @@ export default function MobileMenu({
<div className="fixed inset-0 bg-black/50 z-[70]" onClick={onClose} />
{/* Menu panel */}
<div className="mobile-menu-panel fixed top-0 right-0 bottom-0 w-64 bg-navy-900 text-white z-[80] flex flex-col shadow-xl">
<div className="flex items-center justify-between px-3 h-11 border-b border-navy-700">
<div className="flex items-center justify-between px-3 h-12 border-b border-navy-700">
<span className="font-semibold">{t('mobileMenu.menu')}</span>
<button
onClick={onClose}

View file

@ -206,7 +206,6 @@ export function useHexagonSelection({
const params = new URLSearchParams({
h3,
resolution: res.toString(),
limit: '100',
offset: offset.toString(),
});
@ -250,7 +249,6 @@ export function useHexagonSelection({
try {
const params = new URLSearchParams({
postcode,
limit: '100',
offset: offset.toString(),
});
if (focusAddress && offset === 0) {

View file

@ -127,7 +127,7 @@ export function useLocationSearch(mode?: string) {
const controller = new AbortController();
abortRef.current = controller;
try {
const params = new URLSearchParams({ q: trimmed, limit: '20' });
const params = new URLSearchParams({ q: trimmed });
if (mode) params.set('mode', mode);
const res = await fetch(
`/api/places?${params}`,

View file

@ -0,0 +1,59 @@
import { useEffect, useState } from 'react';
import type { NearbyStation, GeoPoint } from '../lib/nearby-stations';
import {
STATION_CATEGORIES,
selectNearbyStations,
stationSearchBounds,
} from '../lib/nearby-stations';
import type { POIResponse } from '../types';
import { apiUrl, assertOk, authHeaders, logNonAbortError } from '../lib/api';
function boundsParam(bounds: ReturnType<typeof stationSearchBounds>): string {
return `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`;
}
export function useNearbyStations(origin: GeoPoint | null) {
const [stations, setStations] = useState<NearbyStation[]>([]);
const [loading, setLoading] = useState(false);
const lat = origin?.lat;
const lon = origin?.lon;
useEffect(() => {
if (lat == null || lon == null) {
setStations([]);
setLoading(false);
return;
}
const controller = new AbortController();
setStations([]);
setLoading(true);
const originPoint = { lat, lon };
const bounds = stationSearchBounds(originPoint);
const params = new URLSearchParams({
bounds: boundsParam(bounds),
categories: STATION_CATEGORIES.join(','),
});
fetch(apiUrl('pois', params), authHeaders({ signal: controller.signal }))
.then((response) => {
assertOk(response, 'nearby stations');
return response.json() as Promise<POIResponse>;
})
.then((json) => {
if (!controller.signal.aborted) {
setStations(selectNearbyStations(json.pois ?? [], originPoint));
}
})
.catch((error) => logNonAbortError('Failed to fetch nearby stations', error))
.finally(() => {
if (!controller.signal.aborted) setLoading(false);
});
return () => controller.abort();
}, [lat, lon]);
return { stations, loading };
}

View file

@ -44,7 +44,7 @@ function getPoiIconUrlForPoi(poi: POI): string {
}
function isBundledPoiIcon(url: string): boolean {
return url.startsWith('/assets/poi-icons/');
return url.startsWith('/assets/poi-icons/') || url.startsWith('data:image/svg+xml');
}
function hasBundledPoiLogo(poi: POI): boolean {

View file

@ -0,0 +1,80 @@
import { cleanup, fireEvent, render } from '@testing-library/react';
import { afterEach, describe, expect, it } from 'vitest';
import type { MutableRefObject } from 'react';
import { useRetainedScrollTop } from './useRetainedScrollTop';
function ScrollPane({
restoreKey,
savedScrollTopRef,
suspendSave,
}: {
restoreKey: string;
savedScrollTopRef: MutableRefObject<number>;
suspendSave: boolean;
}) {
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey,
scrollTopRef: savedScrollTopRef,
suspendSave,
});
return (
<div ref={scrollRef} onScroll={onScroll} data-testid="pane">
<div style={{ height: 2000 }} />
</div>
);
}
describe('useRetainedScrollTop', () => {
afterEach(() => {
cleanup();
});
it('keeps the saved scroll offset while replacement content is loading', () => {
const savedScrollTopRef = { current: 0 };
const view = render(
<ScrollPane
restoreKey="area:a"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
const pane = view.getByTestId('pane');
pane.scrollTop = 360;
fireEvent.scroll(pane);
expect(savedScrollTopRef.current).toBe(360);
view.rerender(
<ScrollPane restoreKey="area:b" savedScrollTopRef={savedScrollTopRef} suspendSave />
);
pane.scrollTop = 0;
fireEvent.scroll(pane);
expect(savedScrollTopRef.current).toBe(360);
view.rerender(
<ScrollPane
restoreKey="area:b"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
expect(pane.scrollTop).toBe(360);
});
it('restores the saved offset when a pane remounts', () => {
const savedScrollTopRef = { current: 220 };
const view = render(
<ScrollPane
restoreKey="area:a"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
expect(view.getByTestId('pane').scrollTop).toBe(220);
});
});

View file

@ -0,0 +1,63 @@
import { useCallback, useLayoutEffect, useRef, type MutableRefObject, type UIEvent } from 'react';
interface UseRetainedScrollTopOptions {
restoreKey: string | null;
scrollTopRef?: MutableRefObject<number>;
suspendSave?: boolean;
}
export function useRetainedScrollTop<T extends HTMLElement>({
restoreKey,
scrollTopRef,
suspendSave = false,
}: UseRetainedScrollTopOptions) {
const fallbackScrollTopRef = useRef(0);
const savedScrollTopRef = scrollTopRef ?? fallbackScrollTopRef;
const nodeRef = useRef<T | null>(null);
const previousRestoreKeyRef = useRef(restoreKey);
const pendingRestoreTopRef = useRef<number | null>(null);
const suspendSaveRef = useRef(suspendSave);
suspendSaveRef.current = suspendSave;
const scrollRef = useCallback(
(node: T | null) => {
const previousNode = nodeRef.current;
if (!node && previousNode && !suspendSaveRef.current) {
savedScrollTopRef.current = previousNode.scrollTop;
}
nodeRef.current = node;
if (node) {
node.scrollTop = pendingRestoreTopRef.current ?? savedScrollTopRef.current;
}
},
[savedScrollTopRef]
);
const onScroll = useCallback(
(event: UIEvent<T>) => {
if (suspendSaveRef.current) return;
savedScrollTopRef.current = event.currentTarget.scrollTop;
},
[savedScrollTopRef]
);
useLayoutEffect(() => {
if (previousRestoreKeyRef.current !== restoreKey) {
previousRestoreKeyRef.current = restoreKey;
pendingRestoreTopRef.current = savedScrollTopRef.current;
}
const node = nodeRef.current;
const pendingRestoreTop = pendingRestoreTopRef.current;
if (!node || pendingRestoreTop == null) return;
node.scrollTop = pendingRestoreTop;
if (!suspendSave) {
pendingRestoreTopRef.current = null;
}
}, [restoreKey, savedScrollTopRef, suspendSave]);
return { scrollRef, onScroll };
}

View file

@ -842,6 +842,8 @@ const de: Translations = {
showAllStatsFallback:
'Wechseln Sie zu allen Immobilien, um dieses Gebiet ohne aktive Filter zu prüfen.',
showAllStats: 'Alle Immobilien anzeigen',
closestStations: 'Nächste Bahnhöfe',
noNearbyStations: 'Keine Bahn- oder U-Bahn-Station im Umkreis von 2 km',
closestBlockingFilters: 'Nächste Änderungen, um dieses Gebiet einzuschließen',
lowerMinTo: 'Minimum auf {{value}} senken',
raiseMaxTo: 'Maximum auf {{value}} erhöhen',

View file

@ -816,6 +816,8 @@ const en = {
showAllStatsFallback:
'Switch to all properties to inspect this area without the active filters.',
showAllStats: 'Show all properties',
closestStations: 'Closest stations',
noNearbyStations: 'No train or tube stations within 2km',
closestBlockingFilters: 'Closest changes to include this area',
lowerMinTo: 'Lower minimum to {{value}}',
raiseMaxTo: 'Raise maximum to {{value}}',

View file

@ -848,6 +848,8 @@ const fr: Translations = {
showAllStatsFallback:
'Passez à toutes les propriétés pour inspecter cette zone sans les filtres actifs.',
showAllStats: 'Afficher toutes les propriétés',
closestStations: 'Stations les plus proches',
noNearbyStations: 'Aucune gare ou station de métro à moins de 2 km',
closestBlockingFilters: 'Modifications les plus proches pour inclure cette zone',
lowerMinTo: 'Abaisser le minimum à {{value}}',
raiseMaxTo: 'Augmenter le maximum à {{value}}',

View file

@ -806,6 +806,8 @@ const hi: Translations = {
showAllStatsFallback:
'सक्रिय फिल्टर के बिना इस क्षेत्र को देखने के लिए सभी संपत्तियों पर जाएं.',
showAllStats: 'सभी संपत्तियां दिखाएं',
closestStations: 'निकटतम स्टेशन',
noNearbyStations: '2 किमी के भीतर कोई ट्रेन या ट्यूब स्टेशन नहीं',
closestBlockingFilters: 'इस क्षेत्र को शामिल करने के निकटतम बदलाव',
lowerMinTo: 'न्यूनतम को {{value}} तक घटाएं',
raiseMaxTo: 'अधिकतम को {{value}} तक बढ़ाएं',

View file

@ -830,6 +830,8 @@ const hu: Translations = {
showAllStatsFallback:
'Váltson az összes ingatlanra, hogy aktív szűrők nélkül tekintse át ezt a területet.',
showAllStats: 'Összes ingatlan mutatása',
closestStations: 'Legközelebbi állomások',
noNearbyStations: 'Nincs vonat- vagy metróállomás 2 km-en belül',
closestBlockingFilters: 'A terület bevonásához legközelebbi módosítások',
lowerMinTo: 'Minimum csökkentése erre: {{value}}',
raiseMaxTo: 'Maximum növelése erre: {{value}}',

View file

@ -775,6 +775,8 @@ const zh: Translations = {
showAllStatsHint: '筛选前这里有 {{count}} 处房产。切换到全部房产即可查看该区域。',
showAllStatsFallback: '切换到全部房产即可在不应用当前筛选条件的情况下查看该区域。',
showAllStats: '显示全部房产',
closestStations: '最近的车站',
noNearbyStations: '2 公里内没有火车站或地铁站',
closestBlockingFilters: '纳入该区域所需的最小调整',
lowerMinTo: '将最小值降至 {{value}}',
raiseMaxTo: '将最大值提高至 {{value}}',

View file

@ -28,6 +28,17 @@ button:not(:disabled),
cursor: pointer;
}
.area-pane-group-header {
box-shadow: inset 0 -1px 0 #e7e5e4;
transition:
background-color 0.2s ease,
color 0.2s ease;
}
html.dark .area-pane-group-header {
box-shadow: inset 0 -1px 0 #1e2d50;
}
/* Smooth theme transitions (scoped to avoid map performance issues) */
body,
div,

View file

@ -132,6 +132,7 @@ export const POI_GROUP_COLORS: Record<string, [number, number, number]> = {
export const POI_CATEGORY_LOGOS: Record<string, string> = {
Airport: '/assets/twemoji/2708.png',
Aldi: '/assets/poi-icons/logos/aldi.svg',
'Allendale Co-operative Society': '/assets/poi-icons/logos/coop.svg',
Amazon: '/assets/poi-icons/brands_2024/amazon_fresh.svg',
Asda: '/assets/poi-icons/logos/asda.svg',
'Asda Express': '/assets/poi-icons/logos/asda.svg',
@ -147,18 +148,26 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'Bus stop': '/assets/twemoji/1f68f.png',
'Butcher & Fishmonger': '/assets/twemoji/1f969.png',
Centra: '/assets/poi-icons/logos/centra.svg',
'Central England Co-operative': '/assets/poi-icons/logos/coop.svg',
'Chelmsford Star Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'Clydebank Co-operative': '/assets/poi-icons/logos/coop.svg',
'Co-op': '/assets/poi-icons/logos/coop.svg',
'Coniston Co-operative Society': '/assets/poi-icons/logos/coop.svg',
COOK: '/assets/poi-icons/brands_2024/cook.svg',
'Convenience Store': '/assets/twemoji/1f3ea.png',
Costco: '/assets/poi-icons/logos/costco.svg',
'Deli & Specialty': '/assets/twemoji/1f9c6.png',
'Dunnes Stores': '/assets/poi-icons/brands_2024/dunnes_stores.svg',
'East of England Co-operative': '/assets/poi-icons/logos/coop.svg',
Farmfoods: '/assets/poi-icons/brands_2023/supermarkets/farmfoods.svg',
Ferry: '/assets/twemoji/26f4.png',
Greengrocer: '/assets/twemoji/1f96c.png',
'Heart of England Co-operative': '/assets/poi-icons/logos/coop.svg',
'Heron Foods': '/assets/poi-icons/brands_2023/supermarkets/heron_foods.svg',
Iceland: '/assets/poi-icons/brands_2024/iceland.svg',
Lidl: '/assets/poi-icons/logos/lidl.svg',
'Langdale Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'Lincolnshire Co-operative': '/assets/poi-icons/logos/coop.svg',
Makro: '/assets/poi-icons/brands_2024/makro.svg',
'M&S': '/assets/poi-icons/brands_2024/mns.svg',
'M&S Clothing': '/assets/poi-icons/brands_2024/mns.svg',
@ -166,6 +175,7 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'M&S Hospital': '/assets/poi-icons/brands_2024/mns.svg',
'M&S MSA': '/assets/poi-icons/brands_2024/mns.svg',
'M&S Outlet': '/assets/poi-icons/brands_2024/mns.svg',
'Midcounties Co-operative': '/assets/poi-icons/logos/coop.svg',
Morrisons: '/assets/poi-icons/logos/morrisons.svg',
'Morrisons Daily': '/assets/poi-icons/brands_2024/morrisons_daily.svg',
'Off-Licence': '/assets/twemoji/1f377.png',
@ -173,12 +183,16 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'Rail station': '/assets/twemoji/1f686.png',
"Sainsbury's": '/assets/poi-icons/logos/sainsburys.svg',
"Sainsbury's Local": '/assets/poi-icons/brands_2024/sainsburys_local.svg',
'Scottish Midland Co-operative': '/assets/poi-icons/logos/coop.svg',
Spar: '/assets/poi-icons/logos/spar.svg',
Supermarket: '/assets/twemoji/1f6d2.png',
'Tamworth Co-operative Society': '/assets/poi-icons/logos/coop.svg',
Tesco: '/assets/poi-icons/logos/tesco.svg',
'Tesco Express': '/assets/poi-icons/logos/tesco_express.svg',
'Tesco Extra': '/assets/poi-icons/logos/tesco_extra.svg',
'Taxi rank': '/assets/twemoji/1f695.png',
'The Radstock Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'The Southern Co-operative': '/assets/poi-icons/logos/coop.svg',
'The Food Warehouse': '/assets/poi-icons/logos/the_food_warehouse.png',
'Tube station': '/assets/poi-icons/public_transport/london_tube.svg',
Waitrose: '/assets/poi-icons/logos/waitrose.svg',

View file

@ -71,7 +71,7 @@ describe('map utilities', () => {
expect(enumIndexToColor(ENUM_PALETTE.length, ENUM_PALETTE)).toEqual(ENUM_PALETTE[0]);
});
it('resolves POI category logos and rejects unknown icon categories', () => {
it('resolves POI category logos and generates a fallback for unknown chains', () => {
expect(getPoiIconUrl('Waitrose', '🛒')).toBe('/assets/poi-icons/logos/waitrose.svg');
expect(getPoiIconUrl('Iceland', '🛒', 'The Food Warehouse')).toBe(
'/assets/poi-icons/logos/the_food_warehouse.png'
@ -83,8 +83,8 @@ describe('map utilities', () => {
expect(getPoiIconUrl('M&S', '🛒', undefined, 'M&S Simply Food')).toBe(
'/assets/poi-icons/visuals/mns.svg'
);
expect(() => getPoiIconUrl('Unknown category', '🛒')).toThrow(
"Missing POI icon for category 'Unknown category'"
expect(getPoiIconUrl('Tian Tian', '🛒')).toMatch(
/^data:image\/svg\+xml;charset=utf-8,/
);
});

View file

@ -309,9 +309,67 @@ function inferPoiIconCategory(category: string, name?: string): string | undefin
}
}
export function getPoiIconUrl(
const GENERATED_POI_LOGO_COLORS: [string, string][] = [
['#0f766e', '#ccfbf1'],
['#1d4ed8', '#dbeafe'],
['#b45309', '#fef3c7'],
['#be123c', '#ffe4e6'],
['#6d28d9', '#ede9fe'],
['#047857', '#d1fae5'],
['#9d174d', '#fce7f3'],
['#334155', '#e2e8f0'],
];
const generatedPoiLogoCache = new Map<string, string>();
function hashLabel(label: string): number {
let hash = 0;
for (let i = 0; i < label.length; i += 1) {
hash = (hash * 31 + label.charCodeAt(i)) >>> 0;
}
return hash;
}
function escapeSvgText(text: string): string {
return text
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;');
}
function getPoiLogoInitials(label: string): string {
const words = label.match(/[A-Za-z0-9]+/g) ?? [];
const significantWords = words.filter(
(word) => !['and', 'of', 'the'].includes(word.toLowerCase())
);
const selectedWords = significantWords.length > 0 ? significantWords : words;
if (selectedWords.length === 0) return 'POI';
if (selectedWords.length === 1) return selectedWords[0].slice(0, 3).toUpperCase();
return selectedWords
.slice(0, 3)
.map((word) => word[0])
.join('')
.toUpperCase();
}
function getGeneratedPoiLogoUrl(label: string): string {
const key = label.trim() || 'POI';
const cached = generatedPoiLogoCache.get(key);
if (cached) return cached;
const [background, foreground] = GENERATED_POI_LOGO_COLORS[
hashLabel(key) % GENERATED_POI_LOGO_COLORS.length
];
const initials = escapeSvgText(getPoiLogoInitials(key));
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="256" height="256" viewBox="0 0 256 256"><rect width="256" height="256" rx="48" fill="${background}"/><text x="128" y="144" text-anchor="middle" font-family="Inter,Arial,sans-serif" font-size="82" font-weight="800" fill="${foreground}">${initials}</text></svg>`;
const url = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
generatedPoiLogoCache.set(key, url);
return url;
}
export function getPoiCategoryLogoUrl(
category: string,
_emoji: string,
iconCategory?: string,
name?: string
): string {
@ -319,11 +377,16 @@ export function getPoiIconUrl(
if (resolvedIconCategory && POI_CATEGORY_LOGOS[resolvedIconCategory]) {
return POI_CATEGORY_LOGOS[resolvedIconCategory];
}
const categoryLogo = POI_CATEGORY_LOGOS[category];
if (!categoryLogo) {
throw new Error(`Missing POI icon for category '${category}'`);
}
return categoryLogo;
return POI_CATEGORY_LOGOS[category] ?? getGeneratedPoiLogoUrl(resolvedIconCategory || category);
}
export function getPoiIconUrl(
category: string,
_emoji: string,
iconCategory?: string,
name?: string
): string {
return getPoiCategoryLogoUrl(category, iconCategory, name);
}
/** Look up a discrete color from the enum palette by index (wraps if > palette size). */

View file

@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest';
import type { POI } from '../types';
import {
formatStationDistance,
selectNearbyStations,
stationSearchBounds,
} from './nearby-stations';
function poi(name: string, lat: number, lng: number): POI {
return {
id: name,
name,
category: 'Rail station',
icon_category: 'Rail station',
group: 'Public Transport',
lat,
lng,
emoji: '',
};
}
describe('selectNearbyStations', () => {
const origin = { lat: 51.5, lon: -0.1 };
it('returns every station within 1km when any station is inside 1km', () => {
const stations = selectNearbyStations(
[poi('Close A', 51.501, -0.1), poi('Close B', 51.502, -0.1), poi('Fallback', 51.511, -0.1)],
origin
);
expect(stations.map((station) => station.name)).toEqual(['Close A', 'Close B']);
});
it('falls back to the nearest three stations within 2km when none are inside 1km', () => {
const stations = selectNearbyStations(
[
poi('Fourth', 51.516, -0.1),
poi('First', 51.51, -0.1),
poi('Outside', 51.53, -0.1),
poi('Third', 51.514, -0.1),
poi('Second', 51.512, -0.1),
],
origin
);
expect(stations.map((station) => station.name)).toEqual(['First', 'Second', 'Third']);
});
it('formats sub-kilometre and kilometre distances', () => {
expect(formatStationDistance(0.234)).toBe('234m');
expect(formatStationDistance(1.234)).toBe('1.2km');
});
});
describe('stationSearchBounds', () => {
it('builds a box around the origin', () => {
const bounds = stationSearchBounds({ lat: 51.5, lon: -0.1 });
expect(bounds.south).toBeLessThan(51.5);
expect(bounds.north).toBeGreaterThan(51.5);
expect(bounds.west).toBeLessThan(-0.1);
expect(bounds.east).toBeGreaterThan(-0.1);
});
});

View file

@ -0,0 +1,71 @@
import type { Bounds, POI } from '../types';
export const STATION_CATEGORIES = ['Rail station', 'Tube station'] as const;
export const STATION_SEARCH_RADIUS_KM = 2;
const PRIMARY_STATION_RADIUS_KM = 1;
const FALLBACK_STATION_LIMIT = 3;
const EARTH_RADIUS_KM = 6371;
const KM_PER_DEGREE_LAT = 111.32;
export interface NearbyStation extends POI {
distanceKm: number;
}
export interface GeoPoint {
lat: number;
lon: number;
}
function toRadians(value: number): number {
return (value * Math.PI) / 180;
}
export function distanceKm(a: GeoPoint, b: GeoPoint): number {
const dLat = toRadians(b.lat - a.lat);
const dLon = toRadians(b.lon - a.lon);
const lat1 = toRadians(a.lat);
const lat2 = toRadians(b.lat);
const h = Math.sin(dLat / 2) ** 2 + Math.cos(lat1) * Math.cos(lat2) * Math.sin(dLon / 2) ** 2;
return 2 * EARTH_RADIUS_KM * Math.asin(Math.sqrt(h));
}
export function stationSearchBounds(origin: GeoPoint, radiusKm = STATION_SEARCH_RADIUS_KM): Bounds {
const latDelta = radiusKm / KM_PER_DEGREE_LAT;
const cosLat = Math.max(Math.cos(toRadians(origin.lat)), 0.01);
const lonDelta = radiusKm / (KM_PER_DEGREE_LAT * cosLat);
return {
south: origin.lat - latDelta,
west: origin.lon - lonDelta,
north: origin.lat + latDelta,
east: origin.lon + lonDelta,
};
}
export function selectNearbyStations(pois: POI[], origin: GeoPoint): NearbyStation[] {
const stations = pois
.map((poi) => ({
...poi,
distanceKm: distanceKm(origin, { lat: poi.lat, lon: poi.lng }),
}))
.filter((poi) => poi.distanceKm <= STATION_SEARCH_RADIUS_KM)
.sort((a, b) => a.distanceKm - b.distanceKm || a.name.localeCompare(b.name));
const withinPrimaryRadius = stations.filter(
(station) => station.distanceKm <= PRIMARY_STATION_RADIUS_KM
);
return withinPrimaryRadius.length > 0
? withinPrimaryRadius
: stations.slice(0, FALLBACK_STATION_LIMIT);
}
export function formatStationDistance(distanceKmValue: number): string {
if (distanceKmValue < 1) {
return `${Math.round(distanceKmValue * 1000)}m`;
}
return `${distanceKmValue.toFixed(1)}km`;
}

View file

@ -202,7 +202,6 @@ export interface Property {
export interface PropertyListResponse {
properties: Property[];
total: number;
limit: number;
offset: number;
truncated: boolean;
}

View file

@ -15,7 +15,7 @@ def test_transform_grocery_retail_points_outputs_chain_categories():
}
)
pois = transform_grocery_retail_points(raw)
pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select(
"id", "name", "category", "icon_category", "group", "emoji"
@ -69,7 +69,7 @@ def test_transform_grocery_retail_points_keeps_fascia_icon_category():
}
)
pois = transform_grocery_retail_points(raw)
pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select("category", "icon_category").to_dicts() == [
{"category": "Tesco", "icon_category": "Tesco Express"},
@ -96,7 +96,7 @@ def test_transform_grocery_retail_points_accepts_base_fascias():
}
)
pois = transform_grocery_retail_points(raw)
pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select("category", "icon_category").to_dicts() == [
{"category": "Aldi", "icon_category": "Aldi"},
@ -118,6 +118,29 @@ def test_transform_grocery_retail_points_drops_invalid_rows():
}
)
pois = transform_grocery_retail_points(raw)
pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois["category"].to_list() == ["Waitrose"]
def test_transform_grocery_retail_points_includes_unmapped_chains_with_five_locations():
raw = pl.DataFrame(
{
"id": list(range(1, 10)),
"retailer": ["Tian Tian"] * 5 + ["Corner Shop"] * 4,
"fascia": ["Tian Tian Market"] * 5 + ["Corner Shop"] * 4,
"store_name": [f"Store {i}" for i in range(1, 10)],
"long_wgs": [-0.1] * 9,
"lat_wgs": [51.5] * 9,
}
)
pois = transform_grocery_retail_points(raw)
assert pois.select("id", "category", "icon_category").to_dicts() == [
{"id": "glx-1", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-2", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-3", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-4", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-5", "category": "Tian Tian", "icon_category": "Tian Tian"},
]

View file

@ -5,7 +5,6 @@ import polars as pl
from pipeline.utils.england_geometry import in_england_mask
DROP_CATEGORIES = {
# Street furniture & infrastructure
"amenity/advice",
@ -1165,49 +1164,44 @@ COOP_RETAILERS = {
"The Southern Co-operative",
}
GROCERY_RETAILER_DISPLAY_NAMES: dict[str, str] = {
"Aldi": "Aldi",
"Asda": "Asda",
"Booths": "Booths",
"Budgens": "Budgens",
"Centra": "Centra",
MIN_GROCERY_CHAIN_LOCATIONS = 5
GROCERY_RETAILER_DISPLAY_NAME_OVERRIDES: dict[str, str] = {
"Cook": "COOK",
"Costco": "Costco",
"Dunnes Stores": "Dunnes Stores",
"Farmfoods": "Farmfoods",
"Heron": "Heron Foods",
"Iceland": "Iceland",
"Lidl": "Lidl",
"Makro": "Makro",
"Marks and Spencer": "M&S",
"Morrisons": "Morrisons",
"Planet Organic": "Planet Organic",
"Sainsburys": "Sainsbury's",
"Spar": "Spar",
"Tesco": "Tesco",
"Waitrose": "Waitrose",
"Whole Foods Market": "Whole Foods Market",
**{retailer: "Co-op" for retailer in COOP_RETAILERS},
"The Co-operative Group": "Co-op",
}
GROCERY_FASCIA_ICON_NAMES: dict[str, str] = {
**GROCERY_RETAILER_DISPLAY_NAMES,
"Aldi": "Aldi",
"Aldi Local": "Aldi",
"Asda": "Asda",
"Asda Express": "Asda Express",
"Asda Living": "Asda Living",
"Asda PFS": "Asda PFS",
"Asda PFS": "Asda",
"Asda Supercentre": "Asda Supercentre",
"Asda Supermarket": "Asda Supermarket",
"Asda Superstore": "Asda Superstore",
"Booths": "Booths",
"Budgens": "Budgens",
"Centra": "Centra",
"Cooltrader": "Heron Foods",
"Co-op Food": "Co-op",
"Cook": "COOK",
"Costco": "Costco",
"Dunnes Stores": "Dunnes Stores",
"Eurospar": "Spar",
"Eurospar PFS": "Spar",
"Farmfoods": "Farmfoods",
"Heron": "Heron Foods",
"Iceland": "Iceland",
"Lidl": "Lidl",
"Little Waitrose": "Little Waitrose",
"Little Waitrose Shell": "Little Waitrose",
"Makro": "Makro",
"Marks and Spencer": "M&S",
"Marks and Spencer BP": "M&S Food",
"Marks and Spencer Clothing": "M&S Clothing",
@ -1221,41 +1215,44 @@ GROCERY_FASCIA_ICON_NAMES: dict[str, str] = {
"Marks and Spencer Travel SF": "M&S Food",
"Morrisons Daily": "Morrisons Daily",
"Morrisons Select": "Morrisons",
"Planet Organic": "Planet Organic",
"Sainsbury's Local": "Sainsbury's Local",
"Sainsburys": "Sainsbury's",
"Sainsburys Local": "Sainsbury's Local",
"Spar": "Spar",
"Spar PFS": "Spar",
"Tesco": "Tesco",
"Tesco Express": "Tesco Express",
"Tesco Express Esso": "Tesco Express",
"Tesco Extra": "Tesco Extra",
"The Co-operative Food": "Co-op",
"The Co-operative Food PFS": "Co-op",
"The Food Warehouse": "The Food Warehouse",
"Waitrose": "Waitrose",
"Waitrose MSA": "Waitrose",
"Whole Foods Market": "Whole Foods Market",
}
def normalize_grocery_retailer(retailer: str | None) -> str:
if retailer is None:
return ""
display_name = GROCERY_RETAILER_DISPLAY_NAMES.get(retailer)
if display_name is None:
raise ValueError(f"Missing grocery retailer display name for {retailer!r}")
return display_name
retailer = retailer.strip()
return GROCERY_RETAILER_DISPLAY_NAME_OVERRIDES.get(retailer, retailer)
def normalize_grocery_icon_category(fascia: str | None, retailer: str | None) -> str:
if fascia:
icon_name = GROCERY_FASCIA_ICON_NAMES.get(fascia)
if icon_name is None:
raise ValueError(f"Missing grocery fascia icon name for {fascia!r}")
return icon_name
icon_name = GROCERY_FASCIA_ICON_NAMES.get(fascia.strip())
if icon_name is not None:
return icon_name
return normalize_grocery_retailer(retailer)
def transform_grocery_retail_points(
grocery_df: pl.DataFrame,
boundary_path: Path | None = None,
min_chain_locations: int = MIN_GROCERY_CHAIN_LOCATIONS,
) -> pl.DataFrame:
"""Convert GEOLYTIX Grocery Retail Points into the POI parquet schema."""
required = {"id", "retailer", "fascia", "store_name", "long_wgs", "lat_wgs"}
@ -1272,6 +1269,11 @@ def transform_grocery_retail_points(
pl.col("lat_wgs").cast(pl.Float64).alias("lat"),
pl.col("long_wgs").cast(pl.Float64).alias("lng"),
)
.with_columns(
pl.col("retailer").str.strip_chars(),
pl.col("fascia").str.strip_chars(),
pl.col("store_name").str.strip_chars(),
)
.drop_nulls(["id", "retailer", "lat", "lng"])
.filter(pl.col("retailer").str.len_chars() > 0)
)
@ -1284,6 +1286,14 @@ def transform_grocery_retail_points(
)
df = df.filter(pl.Series(mask))
eligible_retailers = (
df.group_by("retailer")
.len()
.filter(pl.col("len") >= min_chain_locations)
.select("retailer")
)
df = df.join(eligible_retailers, on="retailer", how="semi")
return df.with_columns(
pl.concat_str([pl.lit("glx-"), pl.col("id")]).alias("id"),
pl.coalesce(["store_name", "fascia", "retailer"])

2
property-data2/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

View file

@ -13,12 +13,10 @@ pub const GRID_CELL_SIZE: f32 = 0.01;
pub const MAX_CELLS_PER_REQUEST: usize = 200000;
pub const MAX_POIS_PER_REQUEST: usize = 3000;
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
pub const DEFAULT_ACTUAL_LISTINGS_LIMIT: usize = 500;
pub const MAX_ACTUAL_LISTINGS_LIMIT: usize = 2000;
pub const MAX_PLACES_LIMIT: usize = 20;
pub const DEFAULT_PLACES_LIMIT: usize = 7;
pub const MAX_PRICE_HISTORY_POINTS: usize = 5000;
pub const PROPERTIES_LIMIT: usize = 100;
pub const ACTUAL_LISTINGS_LIMIT: usize = 500;
pub const PLACES_LIMIT: usize = 20;
pub const PRICE_HISTORY_POINTS_LIMIT: usize = 5000;
pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02;
pub const AI_FILTERS_MAX_TOKENS: usize = 2000;

View file

@ -30,6 +30,16 @@ const GROCERY_DASHBOARD_CATEGORIES: &[&str] = &[
"Budgens",
"Centra",
"Co-op",
"Central England Co-operative",
"Chelmsford Star Co-operative Society",
"East of England Co-operative",
"Heart of England Co-operative",
"Lincolnshire Co-operative",
"Midcounties Co-operative",
"Scottish Midland Co-operative",
"Tamworth Co-operative Society",
"The Radstock Co-operative Society",
"The Southern Co-operative",
"COOK",
"Costco",
"Dunnes Stores",

View file

@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use tracing::info;
use crate::api_error::ApiError;
use crate::consts::{DEFAULT_ACTUAL_LISTINGS_LIMIT, MAX_ACTUAL_LISTINGS_LIMIT};
use crate::consts::ACTUAL_LISTINGS_LIMIT;
use crate::data::ActualListing;
use crate::features::property_level_feature_names;
use crate::parsing::{
@ -24,9 +24,6 @@ pub struct ActualListingsParams {
filters: Option<String>,
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
travel: Option<String>,
/// Page size — defaults to DEFAULT_ACTUAL_LISTINGS_LIMIT, capped at
/// MAX_ACTUAL_LISTINGS_LIMIT.
limit: Option<usize>,
/// Number of results to skip. Defaults to 0.
offset: Option<usize>,
}
@ -35,7 +32,6 @@ pub struct ActualListingsParams {
pub struct ActualListingsResponse {
pub listings: Vec<ActualListing>,
pub total: usize,
pub limit: usize,
pub offset: usize,
pub truncated: bool,
}
@ -45,16 +41,12 @@ pub async fn get_actual_listings(
Query(params): Query<ActualListingsParams>,
) -> Result<Json<ActualListingsResponse>, ApiError> {
let state = shared.load_state();
let limit = params
.limit
.unwrap_or(DEFAULT_ACTUAL_LISTINGS_LIMIT)
.min(MAX_ACTUAL_LISTINGS_LIMIT);
let limit = ACTUAL_LISTINGS_LIMIT;
let offset = params.offset.unwrap_or(0);
let Some(actual_listings) = state.actual_listings.clone() else {
return Ok(Json(ActualListingsResponse {
listings: Vec::new(),
total: 0,
limit,
offset,
truncated: false,
}));
@ -162,7 +154,6 @@ pub async fn get_actual_listings(
Ok(ActualListingsResponse {
listings,
total: total_matching,
limit,
offset,
truncated,
})

View file

@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
use tracing::info;
use crate::api_error::ApiError;
use crate::consts::{DEFAULT_PLACES_LIMIT, MAX_PLACES_LIMIT};
use crate::consts::PLACES_LIMIT;
use crate::data::{normalize_search_text, slugify};
use crate::state::SharedState;
@ -42,7 +42,6 @@ pub struct PlacesResponse {
#[allow(clippy::min_ident_chars)]
pub struct PlacesParams {
q: String,
limit: Option<usize>,
/// If set, only return places that have travel time data for this mode.
mode: Option<String>,
}
@ -105,10 +104,7 @@ pub async fn get_places(
params.q
};
let limit = params
.limit
.unwrap_or(DEFAULT_PLACES_LIMIT)
.min(MAX_PLACES_LIMIT);
let limit = PLACES_LIMIT;
let mode_filter = params.mode;
let places = tokio::task::spawn_blocking(move || {

View file

@ -8,7 +8,7 @@ use serde::Deserialize;
use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, POSTCODE_SEARCH_OFFSET};
use crate::consts::{POSTCODE_SEARCH_OFFSET, PROPERTIES_LIMIT};
use crate::licensing::{check_license_point, resolve_share_code};
use crate::parsing::{parse_filters_with_poi, row_passes_filters, row_passes_poi_filters};
use crate::state::SharedState;
@ -24,7 +24,6 @@ pub struct PostcodePropertiesParams {
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`.
/// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>,
pub limit: Option<usize>,
pub offset: Option<usize>,
/// Exact address to rank first when opening properties from address search.
pub focus_address: Option<String>,
@ -151,7 +150,7 @@ pub async fn get_postcode_properties(
});
let total = matching_rows.len();
let limit = params.limit.unwrap_or(DEFAULT_PROPERTIES_LIMIT);
let limit = PROPERTIES_LIMIT;
let page_offset = params.offset.unwrap_or(0);
let truncated = total > page_offset + limit;
@ -186,7 +185,6 @@ pub async fn get_postcode_properties(
Ok(PropertyListResponse {
properties,
total,
limit,
offset: page_offset,
truncated,
})

View file

@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
use tracing::{info, warn};
use crate::auth::OptionalUser;
use crate::consts::DEFAULT_PROPERTIES_LIMIT;
use crate::consts::PROPERTIES_LIMIT;
use crate::data::RenovationEvent;
use crate::licensing::{check_license_bounds, resolve_share_code};
use crate::parsing::{
@ -29,7 +29,6 @@ pub struct HexagonPropertiesParams {
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`.
/// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>,
pub limit: Option<usize>,
pub offset: Option<usize>,
/// Share-link code; grants bbox-scoped access for unlicensed users.
pub share: Option<String>,
@ -69,7 +68,6 @@ pub struct Property {
pub struct PropertyListResponse {
pub properties: Vec<Property>,
pub total: usize,
pub limit: usize,
pub offset: usize,
pub truncated: bool,
}
@ -276,7 +274,7 @@ pub async fn get_hexagon_properties(
matching_rows.sort_unstable_by_key(|&row| state.data.address(row).trim().is_empty());
let total = matching_rows.len();
let limit = params.limit.unwrap_or(DEFAULT_PROPERTIES_LIMIT);
let limit = PROPERTIES_LIMIT;
let offset = params.offset.unwrap_or(0);
let truncated = total > offset + limit;
@ -312,7 +310,6 @@ pub async fn get_hexagon_properties(
Ok(PropertyListResponse {
properties,
total,
limit,
offset,
truncated,
})

View file

@ -4,7 +4,7 @@ use metrics::counter;
use rustc_hash::FxHashMap;
use tracing::error;
use crate::consts::MAX_PRICE_HISTORY_POINTS;
use crate::consts::PRICE_HISTORY_POINTS_LIMIT;
use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
use super::hexagon_stats::{EnumFeatureStats, HistogramStats, NumericFeatureStats, PricePoint};
@ -32,9 +32,9 @@ pub fn extract_price_history(
}
})
.collect();
if points.len() > MAX_PRICE_HISTORY_POINTS {
let step = points.len() as f64 / MAX_PRICE_HISTORY_POINTS as f64;
points = (0..MAX_PRICE_HISTORY_POINTS)
if points.len() > PRICE_HISTORY_POINTS_LIMIT {
let step = points.len() as f64 / PRICE_HISTORY_POINTS_LIMIT as f64;
points = (0..PRICE_HISTORY_POINTS_LIMIT)
.map(|i| {
let idx = (i as f64 * step) as usize;
PricePoint {