all is well
Some checks failed
Build and publish Docker image / build-and-push (push) Failing after 7m0s
CI / Check (push) Failing after 7m9s

This commit is contained in:
Andras Schmelczer 2026-05-17 17:20:19 +01:00
parent eac1bd0d13
commit 2f149503bb
53 changed files with 1543 additions and 354 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

View file

@ -82,7 +82,7 @@ PROPERTY_TYPE_MAP = {
"Farm / Barn": "Other", "Farm / Barn": "Other",
"Farm House": "Other", "Farm House": "Other",
"House": "Detached", "House": "Detached",
"House of Multiple Occupation": "Flats/Maisonettes", "House of Multiple Occupation": "Other",
"House Share": "Other", "House Share": "Other",
"Not Specified": "Other", "Not Specified": "Other",
"Chalet": "Other", "Chalet": "Other",
@ -90,15 +90,15 @@ PROPERTY_TYPE_MAP = {
"Coach House": "Other", "Coach House": "Other",
"Character Property": "Other", "Character Property": "Other",
"Cluster House": "Other", "Cluster House": "Other",
"Retirement Property": "Flats/Maisonettes", "Retirement Property": "Other",
"Parking": "Other", "Parking": "Other",
"Plot": "Other", "Plot": "Other",
"Garages": "Other", "Garages": "Other",
"Mews": "Terraced", "Mews": "Terraced",
"Property": "Other", "Property": "Other",
"Flat Share": "Other", "Flat Share": "Other",
"Block of Apartments": "Flats/Maisonettes", "Block of Apartments": "Other",
"Private Halls": "Flats/Maisonettes", "Private Halls": "Other",
"Terraced Bungalow": "Terraced", "Terraced Bungalow": "Terraced",
"Equestrian Facility": "Other", "Equestrian Facility": "Other",
"Ground Maisonette": "Flats/Maisonettes", "Ground Maisonette": "Flats/Maisonettes",
@ -107,13 +107,13 @@ PROPERTY_TYPE_MAP = {
"Farm Land": "Other", "Farm Land": "Other",
"House Boat": "Other", "House Boat": "Other",
"Barn": "Other", "Barn": "Other",
"Serviced Apartments": "Flats/Maisonettes", "Serviced Apartments": "Other",
# Space-separated variants (from home.co.uk underscore/hyphen normalization) # Space-separated variants (from home.co.uk underscore/hyphen normalization)
"Semi Detached": "Semi-Detached", "Semi Detached": "Semi-Detached",
"Semi Detached Bungalow": "Semi-Detached", "Semi Detached Bungalow": "Semi-Detached",
"End Of Terrace": "Terraced", "End Of Terrace": "Terraced",
"End Terrace": "Terraced", "End Terrace": "Terraced",
"Block Of Apartments": "Flats/Maisonettes", "Block Of Apartments": "Other",
# Lowercase variants (from home.co.uk / Rightmove APIs) # Lowercase variants (from home.co.uk / Rightmove APIs)
"house": "Detached", "house": "Detached",
"bungalow": "Other", "bungalow": "Other",
@ -121,7 +121,7 @@ PROPERTY_TYPE_MAP = {
"land": "Other", "land": "Other",
"other": "Other", "other": "Other",
"not-specified": "Other", "not-specified": "Other",
"retirement-property": "Flats/Maisonettes", "retirement-property": "Other",
"equestrian-facility": "Other", "equestrian-facility": "Other",
"flat": "Flats/Maisonettes", "flat": "Flats/Maisonettes",
"detached": "Detached", "detached": "Detached",

View file

@ -19,7 +19,12 @@ from constants import (
RETRY_BASE_DELAY, RETRY_BASE_DELAY,
) )
from spatial import PostcodeSpatialIndex from spatial import PostcodeSpatialIndex
from transform import normalize_postcode, normalize_sub_type, validate_floor_area from transform import (
normalize_postcode,
normalize_sub_type,
parse_int_value,
validate_floor_area,
)
log = logging.getLogger("homecouk") log = logging.getLogger("homecouk")
@ -170,11 +175,19 @@ def parse_floor_area(description: str | None) -> float | None:
"""Try to extract floor area from description text like '789 sq.ft.' or '73 sq.m.'.""" """Try to extract floor area from description text like '789 sq.ft.' or '73 sq.m.'."""
if not description: if not description:
return None return None
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*ft", description, re.IGNORECASE) m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))",
description,
re.IGNORECASE,
)
if m: if m:
sqft = float(m.group(1).replace(",", "")) sqft = float(m.group(1).replace(",", ""))
return validate_floor_area(round(sqft * 0.092903, 1)) return validate_floor_area(round(sqft * 0.092903, 1))
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*m", description, re.IGNORECASE) m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))",
description,
re.IGNORECASE,
)
if m: if m:
return validate_floor_area(round(float(m.group(1).replace(",", "")), 1)) return validate_floor_area(round(float(m.group(1).replace(",", "")), 1))
return None return None
@ -237,6 +250,15 @@ def map_property_type(raw_type: str | None) -> str:
# Home.co.uk uses types like "House", "Flat", "Apartment", "Detached", etc. # Home.co.uk uses types like "House", "Flat", "Apartment", "Detached", etc.
# Try common patterns # Try common patterns
lower = raw_type.lower() lower = raw_type.lower()
excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if ( if (
"flat" in lower "flat" in lower
or "apartment" in lower or "apartment" in lower
@ -269,8 +291,10 @@ def transform_property(
log.debug("Coords outside England: lat=%.4f lng=%.4f — skipping", lat, lng) log.debug("Coords outside England: lat=%.4f lng=%.4f — skipping", lat, lng)
return None return None
price = prop.get("price") or prop.get("latest_price") price = parse_int_value(prop.get("price")) or parse_int_value(
if not price or int(price) <= 0: prop.get("latest_price")
)
if not price or price <= 0:
return None return None
# Home.co.uk provides postcodes directly, but fall back to spatial index # Home.co.uk provides postcodes directly, but fall back to spatial index
@ -281,10 +305,10 @@ def transform_property(
log.debug("No postcode for property at %.4f, %.4f — skipping", lat, lng) log.debug("No postcode for property at %.4f, %.4f — skipping", lat, lng)
return None return None
raw_beds = prop.get("bedrooms", 0) or 0 raw_beds = parse_int_value(prop.get("bedrooms")) or 0
raw_baths = prop.get("bathrooms", 0) or 0 raw_baths = parse_int_value(prop.get("bathrooms")) or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0 bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0 bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS: if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning( log.warning(
"home.co.uk %s: implausible beds=%d baths=%d (capped to 0)", "home.co.uk %s: implausible beds=%d baths=%d (capped to 0)",
@ -318,7 +342,7 @@ def transform_property(
"Leasehold/Freehold": parse_tenure(prop), "Leasehold/Freehold": parse_tenure(prop),
"Property type": map_property_type(listing_type), "Property type": map_property_type(listing_type),
"Property sub-type": normalize_sub_type(listing_type), "Property sub-type": normalize_sub_type(listing_type),
"price": int(price), "price": price,
"price_frequency": "", "price_frequency": "",
"Price qualifier": price_qualifier, "Price qualifier": price_qualifier,
"Total floor area (sqm)": parse_floor_area(prop.get("description")), "Total floor area (sqm)": parse_floor_area(prop.get("description")),
@ -362,7 +386,16 @@ def search_outcode(
break break
for prop in raw_props: for prop in raw_props:
try:
transformed = transform_property(prop, pc_index) transformed = transform_property(prop, pc_index)
except Exception as exc:
log.warning(
"home.co.uk %s property %s failed to transform: %s",
outcode,
prop.get("listing_id") or prop.get("property_id") or "?",
exc,
)
continue
if transformed: if transformed:
properties.append(transformed) properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties: if max_properties is not None and len(properties) >= max_properties:

63
finder/listing_filters.py Normal file
View file

@ -0,0 +1,63 @@
"""Shared target filters for manual buy-listing scrapes."""
import math
from typing import Any
BUY_MAX_PRICE = 1_000_000
BUY_MIN_BEDROOMS = 2
BUY_MAX_BEDROOMS = 5
BUY_ALLOWED_BATHROOMS = frozenset({2, 3})
BUY_MIN_FLOOR_AREA_SQM = 90.0
BUY_MAX_FLOOR_AREA_SQM = 170.0
BUY_PROPERTY_TYPES = frozenset({"Flats/Maisonettes"})
BUY_MIN_FLOOR_AREA_SQFT = round(BUY_MIN_FLOOR_AREA_SQM / 0.092903)
BUY_MAX_FLOOR_AREA_SQFT = round(BUY_MAX_FLOOR_AREA_SQM / 0.092903)
def _number(value: Any) -> float | None:
if value is None:
return None
try:
number = float(value)
except (TypeError, ValueError):
return None
if not math.isfinite(number):
return None
return number
def _int(value: Any) -> int | None:
number = _number(value)
if number is None or not number.is_integer():
return None
return int(number)
def matches_strict_buy_listing_filter(prop: dict) -> bool:
"""Exact filter used to guard scraped/output datasets."""
if "price" in prop:
price = _number(prop.get("price"))
else:
price = _number(prop.get("Asking price"))
if price is None or price <= 0 or price >= BUY_MAX_PRICE:
return False
bedrooms = _int(prop.get("Bedrooms"))
if bedrooms is None or (
bedrooms < BUY_MIN_BEDROOMS or bedrooms > BUY_MAX_BEDROOMS
):
return False
property_type = prop.get("Property type")
if property_type not in BUY_PROPERTY_TYPES:
return False
bathrooms = _int(prop.get("Bathrooms"))
if bathrooms not in BUY_ALLOWED_BATHROOMS:
return False
floor_area = _number(prop.get("Total floor area (sqm)"))
if floor_area is None:
return False
return BUY_MIN_FLOOR_AREA_SQM <= floor_area <= BUY_MAX_FLOOR_AREA_SQM

View file

@ -10,6 +10,15 @@ from constants import (
TYPEAHEAD_URL, TYPEAHEAD_URL,
) )
from http_client import fetch_with_retry from http_client import fetch_with_retry
from listing_filters import (
BUY_ALLOWED_BATHROOMS,
BUY_MAX_BEDROOMS,
BUY_MAX_FLOOR_AREA_SQFT,
BUY_MAX_PRICE,
BUY_MIN_BEDROOMS,
BUY_MIN_FLOOR_AREA_SQFT,
matches_strict_buy_listing_filter,
)
from spatial import PostcodeSpatialIndex from spatial import PostcodeSpatialIndex
from transform import transform_property from transform import transform_property
@ -22,12 +31,23 @@ outcode_cache: dict[str, str] = {}
# Requesting index >= 1008 returns HTTP 400. # Requesting index >= 1008 returns HTTP 400.
_MAX_INDEX = 1008 _MAX_INDEX = 1008
# Property type filters for splitting overcapped searches. Each sub-query _BASE_BUY_SEARCH_PARAMS = {
# gets its own 1008 cap, so we can recover listings beyond the unfiltered limit. "propertyTypes": "flat",
_PROPERTY_TYPES = [ "minBedrooms": str(BUY_MIN_BEDROOMS),
"detached", "semi-detached", "terraced", "flat", "maxBedrooms": str(BUY_MAX_BEDROOMS),
"bungalow", "park-home", "land", "minBathrooms": str(min(BUY_ALLOWED_BATHROOMS)),
] "maxBathrooms": str(max(BUY_ALLOWED_BATHROOMS)),
"minSize": str(BUY_MIN_FLOOR_AREA_SQFT),
"maxSize": str(BUY_MAX_FLOOR_AREA_SQFT),
"maxPrice": str(BUY_MAX_PRICE - 1),
}
def _buy_search_params(extra_params: dict | None = None) -> dict:
params = dict(_BASE_BUY_SEARCH_PARAMS)
if extra_params:
params.update(extra_params)
return params
def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None: def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
@ -92,8 +112,18 @@ def _paginate(
break break
for prop in raw_props: for prop in raw_props:
try:
transformed = transform_property(prop, outcode, pc_index) transformed = transform_property(prop, outcode, pc_index)
if transformed: except Exception as exc:
log.warning(
"Rightmove %s/%s property %s failed to transform: %s",
outcode,
channel_cfg["channel"],
prop.get("id", "?"),
exc,
)
continue
if transformed and matches_strict_buy_listing_filter(transformed):
properties.append(transformed) properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties: if max_properties is not None and len(properties) >= max_properties:
return properties, result_count return properties, result_count
@ -105,6 +135,15 @@ def _paginate(
if index >= result_count: if index >= result_count:
break break
if index >= _MAX_INDEX:
log.warning(
"%s/%s: %d filtered results exceed Rightmove's %d-result page cap",
outcode,
channel_cfg["channel"],
result_count,
_MAX_INDEX,
)
break
time.sleep(DELAY_BETWEEN_PAGES) time.sleep(DELAY_BETWEEN_PAGES)
@ -121,54 +160,20 @@ def search_outcode(
) -> list[dict]: ) -> list[dict]:
"""Paginate through search results for one outcode+channel. Returns transformed properties. """Paginate through search results for one outcode+channel. Returns transformed properties.
When the unfiltered result count exceeds 1008 (Rightmove's hard pagination cap), Search requests set the supported Rightmove filters directly: flats,
re-queries per property type to recover listings beyond the cap. 2-5 bedrooms, 2-3 bathrooms, 969-1830 sq ft, and asking price below £1m.
""" """
properties, result_count = _paginate( properties, _ = _paginate(
client, outcode_id, outcode, channel_cfg, pc_index, max_properties=max_properties client,
outcode_id,
outcode,
channel_cfg,
pc_index,
extra_params=_buy_search_params(),
max_properties=max_properties,
) )
if max_properties is not None and len(properties) >= max_properties: if max_properties is not None and len(properties) >= max_properties:
return properties[:max_properties] return properties[:max_properties]
if result_count <= _MAX_INDEX:
return properties
# Hit the 1008 cap — re-search per property type to get full coverage
ch = channel_cfg["channel"]
log.info(
"%s/%s: %d results exceed %d cap, splitting by property type",
outcode, ch, result_count, _MAX_INDEX,
)
all_by_id: dict[str, dict] = {p["id"]: p for p in properties}
for pt in _PROPERTY_TYPES:
pt_props, _ = _paginate(
client, outcode_id, outcode, channel_cfg, pc_index,
extra_params={"propertyTypes": pt},
max_properties=max_properties,
)
new = 0
for p in pt_props:
if p["id"] not in all_by_id:
all_by_id[p["id"]] = p
new += 1
if (
max_properties is not None
and len(all_by_id) >= max_properties
):
break
if new:
log.debug("%s/%s type=%s: +%d new properties", outcode, ch, pt, new)
if max_properties is not None and len(all_by_id) >= max_properties:
break
log.info(
"%s/%s: type split recovered %d%d properties",
outcode, ch, len(properties), len(all_by_id),
)
properties = list(all_by_id.values())
if max_properties is not None:
return properties[:max_properties]
return properties return properties

View file

@ -19,6 +19,7 @@ from homecouk import load_cookies as load_homecouk_cookies
from homecouk import make_client as make_homecouk_client from homecouk import make_client as make_homecouk_client
from homecouk import search_outcode as homecouk_search_outcode from homecouk import search_outcode as homecouk_search_outcode
from http_client import make_client from http_client import make_client
from listing_filters import matches_strict_buy_listing_filter
from rightmove import resolve_outcode_id from rightmove import resolve_outcode_id
from rightmove import search_outcode as rightmove_search_outcode from rightmove import search_outcode as rightmove_search_outcode
from spatial import PostcodeSpatialIndex from spatial import PostcodeSpatialIndex
@ -181,11 +182,11 @@ def _source_names(sources: str | Iterable[str] | None) -> list[str]:
requested = [str(source).strip().lower() for source in sources] requested = [str(source).strip().lower() for source in sources]
requested = [source for source in requested if source] requested = [source for source in requested if source]
if "all" in requested: unknown = sorted(set(requested) - set(SOURCE_ORDER) - {"all"})
return list(SOURCE_ORDER)
unknown = sorted(set(requested) - set(SOURCE_ORDER))
if unknown: if unknown:
raise ValueError(f"Unknown source(s): {', '.join(unknown)}") raise ValueError(f"Unknown source(s): {', '.join(unknown)}")
if "all" in requested:
return list(SOURCE_ORDER)
return [source for source in SOURCE_ORDER if source in requested] return [source for source in SOURCE_ORDER if source in requested]
@ -196,19 +197,28 @@ def _dedup_key(prop: dict) -> tuple:
def _merge_properties(source_results: dict[str, list[dict]]) -> tuple[list[dict], dict, int]: def _merge_properties(source_results: dict[str, list[dict]]) -> tuple[list[dict], dict, int]:
merged: dict[str, dict] = {} merged: dict[str, dict] = {}
seen_keys: set[tuple] = set() seen_keys: set[tuple] = set()
seen_ids: set[str] = set()
counts = {source: 0 for source in SOURCE_ORDER} counts = {source: 0 for source in SOURCE_ORDER}
deduped = 0 deduped = 0
for source in SOURCE_ORDER: for source in SOURCE_ORDER:
for prop in source_results.get(source, []): for prop in source_results.get(source, []):
prop_id = prop.get("id") prop_id = prop.get("id")
key = _dedup_key(prop) if prop_id is not None:
if (prop_id is not None and prop_id in merged) or key in seen_keys: prop_id = str(prop_id)
if prop_id in seen_ids:
deduped += 1
continue
seen_ids.add(prop_id)
storage_key = prop_id
else:
key = _dedup_key(prop)
if key in seen_keys:
deduped += 1 deduped += 1
continue continue
storage_key = prop_id if prop_id is not None else f"{source}:{len(merged)}"
merged[storage_key] = prop
seen_keys.add(key) seen_keys.add(key)
storage_key = f"{source}:{len(merged)}"
merged[storage_key] = prop
counts[source] += 1 counts[source] += 1
return list(merged.values()), counts, deduped return list(merged.values()), counts, deduped
@ -241,13 +251,22 @@ def _store_properties(
if remaining == 0: if remaining == 0:
return 0 return 0
eligible = [prop for prop in props if _property_is_londonish(prop)] londonish = [prop for prop in props if _property_is_londonish(prop)]
dropped = len(props) - len(eligible) dropped_outside_area = len(props) - len(londonish)
if dropped: if dropped_outside_area:
log.debug( log.debug(
"%s dropped %d properties outside the Greater London-ish postcode filter", "%s dropped %d properties outside the Greater London-ish postcode filter",
source, source,
dropped, dropped_outside_area,
)
eligible = [prop for prop in londonish if matches_strict_buy_listing_filter(prop)]
dropped_non_matching = len(londonish) - len(eligible)
if dropped_non_matching:
log.debug(
"%s dropped %d properties outside the strict buy-listing filters",
source,
dropped_non_matching,
) )
selected = eligible if remaining is None else eligible[:remaining] selected = eligible if remaining is None else eligible[:remaining]
@ -367,20 +386,16 @@ def _scrape_homecouk(
log.info("home.co.uk cap reached") log.info("home.co.uk cap reached")
return return
remaining = _source_remaining(
results, "homecouk", max_properties_per_source
)
if remaining == 0:
log.info("home.co.uk cap reached")
return
for attempt in range(2): for attempt in range(2):
try: try:
# home.co.uk cannot express the full filter set at source.
# Fetch the outcode page set first; _store_properties applies
# the strict filter and source cap after transformation.
props = homecouk_search_outcode( props = homecouk_search_outcode(
client, client,
outcode, outcode,
pc_index, pc_index,
max_properties=remaining, max_properties=None,
) )
added = _store_properties( added = _store_properties(
results, results,
@ -442,19 +457,17 @@ def _scrape_zoopla(
log.info("Zoopla cap reached") log.info("Zoopla cap reached")
return return
remaining = _source_remaining(results, "zoopla", max_properties_per_source)
if remaining == 0:
log.info("Zoopla cap reached")
return
for attempt in range(2): for attempt in range(2):
try: try:
# Zoopla source-side filters are unverified here. Fetch the
# outcode page set first; _store_properties applies the
# strict filter and source cap after transformation.
props, _ = zoopla_search_outcode( props, _ = zoopla_search_outcode(
page, page,
outcode, outcode,
pc_index, pc_index,
pc_coords, pc_coords,
max_properties=remaining, max_properties=None,
) )
added = _store_properties( added = _store_properties(
results, results,
@ -506,9 +519,6 @@ def run_scrape(
output_base = Path(output_dir) if output_dir is not None else DATA_DIR output_base = Path(output_dir) if output_dir is not None else DATA_DIR
output_base.mkdir(parents=True, exist_ok=True) output_base.mkdir(parents=True, exist_ok=True)
if "zoopla" in selected_sources and pc_coords is None:
pc_coords = build_postcode_coords()
errors: list[str] = [] errors: list[str] = []
results = {source: [] for source in SOURCE_ORDER} results = {source: [] for source in SOURCE_ORDER}
started_at = time.time() started_at = time.time()
@ -539,7 +549,8 @@ def run_scrape(
) )
if "zoopla" in selected_sources: if "zoopla" in selected_sources:
assert pc_coords is not None if pc_coords is None:
pc_coords = build_postcode_coords()
_scrape_zoopla( _scrape_zoopla(
selected_outcodes, selected_outcodes,
pc_index, pc_index,
@ -551,19 +562,36 @@ def run_scrape(
merged, source_counts, deduped = _merge_properties(results) merged, source_counts, deduped = _merge_properties(results)
output_path = output_base / "online_listings_buy.parquet" output_path = output_base / "online_listings_buy.parquet"
if merged:
write_parquet(merged, output_path) write_parquet(merged, output_path)
else:
if output_path.exists():
output_path.unlink()
log.warning("No strict properties to write to %s", output_path)
filtered = [prop for prop in merged if matches_strict_buy_listing_filter(prop)]
filtered_output_path = output_base / "online_listings_buy_filtered.parquet"
if filtered:
write_parquet(filtered, filtered_output_path)
else:
if filtered_output_path.exists():
filtered_output_path.unlink()
log.warning("No strict-filtered properties to write to %s", filtered_output_path)
counts = { counts = {
"total": len(merged), "total": len(merged),
"filtered_total": len(filtered),
"deduped": deduped, "deduped": deduped,
"sources": source_counts, "sources": source_counts,
} }
source_summary = " ".join(
f"{source}:{source_counts[source]}" for source in SOURCE_ORDER
)
log.info( log.info(
"Sale scrape complete: %d unique (rightmove:%d homecouk:%d zoopla:%d deduped:%d)", "Sale scrape complete: %d unique, %d strict-filtered (%s deduped:%d)",
len(merged), len(merged),
source_counts["rightmove"], len(filtered),
source_counts["homecouk"], source_summary,
source_counts["zoopla"],
deduped, deduped,
) )
@ -575,6 +603,7 @@ def run_scrape(
}, },
"counts": counts, "counts": counts,
"path": str(output_path), "path": str(output_path),
"filtered_path": str(filtered_output_path),
"errors": errors, "errors": errors,
"elapsed_seconds": round(time.time() - started_at, 3), "elapsed_seconds": round(time.time() - started_at, 3),
} }

View file

@ -45,9 +45,10 @@ def write_parquet(properties: list[dict], path: Path) -> None:
remapped = 0 remapped = 0
for p in properties: for p in properties:
sub_type = p.get("Property sub-type", "") sub_type = p.get("Property sub-type", "")
if sub_type and sub_type != "Unknown": current_type = p.get("Property type")
if sub_type and sub_type != "Unknown" and current_type in (None, "", "Other"):
new_type = map_property_type(sub_type) new_type = map_property_type(sub_type)
if new_type != p.get("Property type"): if new_type != current_type:
p["Property type"] = new_type p["Property type"] = new_type
remapped += 1 remapped += 1
if remapped: if remapped:

View file

@ -1,4 +1,5 @@
import logging import logging
import math
import re import re
from constants import MAX_BEDROOMS, PROPERTY_TYPE_MAP, RIGHTMOVE_BASE from constants import MAX_BEDROOMS, PROPERTY_TYPE_MAP, RIGHTMOVE_BASE
@ -29,17 +30,43 @@ def validate_floor_area(sqm: float | None) -> float | None:
return sqm return sqm
def parse_int_value(value) -> int | None:
"""Parse an integer-like API value without truncating decimals."""
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
if not math.isfinite(value) or not value.is_integer():
return None
return int(value)
if isinstance(value, str):
cleaned = value.strip().replace(",", "").replace("£", "")
if not re.fullmatch(r"\d+", cleaned):
return None
return int(cleaned)
return None
def parse_display_size(display_size: str | None) -> float | None: def parse_display_size(display_size: str | None) -> float | None:
"""Parse displaySize like '499 sq. ft.' or '4,124 sq. ft.' to sqm.""" """Parse displaySize like '499 sq. ft.' or '4,124 sq. ft.' to sqm."""
if not display_size: if not display_size:
return None return None
# Try sq. ft. first # Try sq. ft. first
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*ft", display_size, re.IGNORECASE) m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))",
display_size,
re.IGNORECASE,
)
if m: if m:
sqft = float(m.group(1).replace(",", "")) sqft = float(m.group(1).replace(",", ""))
return validate_floor_area(round(sqft * 0.092903, 1)) return validate_floor_area(round(sqft * 0.092903, 1))
# Try sq. m. # Try sq. m.
m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*m", display_size, re.IGNORECASE) m = re.search(
r"([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))",
display_size,
re.IGNORECASE,
)
if m: if m:
return validate_floor_area(round(float(m.group(1).replace(",", "")), 1)) return validate_floor_area(round(float(m.group(1).replace(",", "")), 1))
return None return None
@ -86,7 +113,21 @@ def map_property_type(sub_type: str | None) -> str:
return canonical return canonical
# Keyword fallback for compound types not in the map # Keyword fallback for compound types not in the map
lower = sub_type.lower() lower = sub_type.lower()
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower: excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
):
return "Flats/Maisonettes" return "Flats/Maisonettes"
if "semi" in lower and "detach" in lower: if "semi" in lower and "detach" in lower:
return "Semi-Detached" return "Semi-Detached"
@ -158,10 +199,10 @@ def transform_property(
lat, lng = fix_coords(raw_lat, raw_lng) lat, lng = fix_coords(raw_lat, raw_lng)
price_obj = prop.get("price", {}) price_obj = prop.get("price", {})
amount = price_obj.get("amount") amount = parse_int_value(price_obj.get("amount"))
if not amount: if not amount:
return None return None
price = int(amount) price = amount
if price <= 0: if price <= 0:
return None return None
@ -172,14 +213,23 @@ def transform_property(
# POA / Auction listings have unreliable prices — treat as no price # POA / Auction listings have unreliable prices — treat as no price
pq_lower = price_qualifier.lower() pq_lower = price_qualifier.lower()
if "poa" in pq_lower or "auction" in pq_lower: non_comparable_price_terms = (
"poa",
"auction",
"shared ownership",
"shared equity",
"part buy",
"part rent",
"from",
)
if any(term in pq_lower for term in non_comparable_price_terms):
return None return None
sub_type = prop.get("propertySubType", "") sub_type = prop.get("propertySubType", "")
raw_beds = prop.get("bedrooms", 0) or 0 raw_beds = parse_int_value(prop.get("bedrooms")) or 0
raw_baths = prop.get("bathrooms", 0) or 0 raw_baths = parse_int_value(prop.get("bathrooms")) or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0 bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0 bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS: if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning( log.warning(
"Rightmove %s: implausible beds=%d baths=%d (capped to 0)", "Rightmove %s: implausible beds=%d baths=%d (capped to 0)",
@ -197,8 +247,15 @@ def transform_property(
log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng) log.debug("No England postcode for property at %.4f, %.4f — skipping", lat, lng)
return None return None
property_url = prop.get("propertyUrl") or ""
if not isinstance(property_url, str):
property_url = ""
listing_id = prop.get("id") or property_url
if not listing_id:
return None
return { return {
"id": prop.get("id"), "id": listing_id,
"Bedrooms": bedrooms, "Bedrooms": bedrooms,
"Bathrooms": bathrooms, "Bathrooms": bathrooms,
"Number of bedrooms & living rooms": bedrooms + bathrooms, "Number of bedrooms & living rooms": bedrooms + bathrooms,
@ -213,7 +270,7 @@ def transform_property(
"price_frequency": "", "price_frequency": "",
"Price qualifier": price_qualifier, "Price qualifier": price_qualifier,
"Total floor area (sqm)": parse_display_size(prop.get("displaySize")), "Total floor area (sqm)": parse_display_size(prop.get("displaySize")),
"Listing URL": RIGHTMOVE_BASE + prop.get("propertyUrl", ""), "Listing URL": RIGHTMOVE_BASE + property_url if property_url else "",
"Listing features": key_features, "Listing features": key_features,
"first_visible_date": prop.get("firstVisibleDate", ""), "first_visible_date": prop.get("firstVisibleDate", ""),
} }

View file

@ -24,7 +24,7 @@ import time
from constants import DELAY_BETWEEN_PAGES, MAX_BEDROOMS, PROPERTY_TYPE_MAP, ZOOPLA_BASE from constants import DELAY_BETWEEN_PAGES, MAX_BEDROOMS, PROPERTY_TYPE_MAP, ZOOPLA_BASE
from spatial import PostcodeSpatialIndex from spatial import PostcodeSpatialIndex
from transform import normalize_sub_type, validate_floor_area from transform import normalize_sub_type, parse_int_value, validate_floor_area
log = logging.getLogger("zoopla") log = logging.getLogger("zoopla")
@ -106,7 +106,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
const bedsMatch = text.match(/(\d+)\s*beds?/i); const bedsMatch = text.match(/(\d+)\s*beds?/i);
const bathsMatch = text.match(/(\d+)\s*baths?/i); const bathsMatch = text.match(/(\d+)\s*baths?/i);
const recMatch = text.match(/(\d+)\s*reception/i); const recMatch = text.match(/(\d+)\s*reception/i);
const areaMatch = text.match(/([\d,]+)\s*sq\.?\s*ft/i); const areaSqftMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))/i);
const areaSqmMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))/i);
let tenure = ''; let tenure = '';
if (/leasehold/i.test(text)) tenure = 'Leasehold'; if (/leasehold/i.test(text)) tenure = 'Leasehold';
@ -141,7 +142,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null, beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null,
baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null, baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null,
receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null, receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null,
floor_area_sqft: areaMatch ? parseInt(areaMatch[1].replace(/,/g, '')) : null, floor_area_sqft: areaSqftMatch ? parseInt(areaSqftMatch[1].replace(/,/g, '')) : null,
floor_area_sqm: areaSqmMatch ? parseFloat(areaSqmMatch[1].replace(/,/g, '')) : null,
address, tenure, property_type, address, tenure, property_type,
}); });
} }
@ -181,7 +183,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
const bedsMatch = text.match(/(\d+)\s*beds?/i); const bedsMatch = text.match(/(\d+)\s*beds?/i);
const bathsMatch = text.match(/(\d+)\s*baths?/i); const bathsMatch = text.match(/(\d+)\s*baths?/i);
const recMatch = text.match(/(\d+)\s*reception/i); const recMatch = text.match(/(\d+)\s*reception/i);
const areaMatch = text.match(/([\d,]+)\s*sq\.?\s*ft/i); const areaSqftMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*ft|square\s+feet|ft(?:\^?2|²))/i);
const areaSqmMatch = text.match(/([\d,]+(?:\.\d+)?)\s*(?:sq\.?\s*m|square\s+met(?:er|re)s?|m(?:\^?2|²))/i);
let address = ''; let address = '';
for (const line of lines) { for (const line of lines) {
@ -225,7 +228,8 @@ _EXTRACT_LISTINGS_JS = r"""() => {
beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null, beds: bedsMatch && parseInt(bedsMatch[1]) <= 20 ? parseInt(bedsMatch[1]) : null,
baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null, baths: bathsMatch && parseInt(bathsMatch[1]) <= 20 ? parseInt(bathsMatch[1]) : null,
receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null, receptions: recMatch && parseInt(recMatch[1]) <= 20 ? parseInt(recMatch[1]) : null,
floor_area_sqft: areaMatch ? parseInt(areaMatch[1].replace(/,/g, '')) : null, floor_area_sqft: areaSqftMatch ? parseInt(areaSqftMatch[1].replace(/,/g, '')) : null,
floor_area_sqm: areaSqmMatch ? parseFloat(areaSqmMatch[1].replace(/,/g, '')) : null,
address, tenure, property_type, address, tenure, property_type,
}); });
} }
@ -611,7 +615,22 @@ def _map_property_type(raw_type: str | None) -> str:
return canonical return canonical
# Keyword fallback # Keyword fallback
lower = raw_type.lower() lower = raw_type.lower()
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower or "penthouse" in lower: excluded_flat_like = (
"block of apartment",
"house of multiple occupation",
"private halls",
"retirement",
"serviced apartment",
)
if any(term in lower for term in excluded_flat_like):
return "Other"
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
or "penthouse" in lower
):
return "Flats/Maisonettes" return "Flats/Maisonettes"
if "semi" in lower and "detach" in lower: if "semi" in lower and "detach" in lower:
return "Semi-Detached" return "Semi-Detached"
@ -634,8 +653,8 @@ def transform_property(
Zoopla search cards do not include coordinates, so we resolve lat/lng Zoopla search cards do not include coordinates, so we resolve lat/lng
from postcodes extracted from the address text.""" from postcodes extracted from the address text."""
price = raw.get("price") price = parse_int_value(raw.get("price"))
if not price or int(price) <= 0: if not price or price <= 0:
return None return None
address = raw.get("address", "") address = raw.get("address", "")
@ -670,10 +689,10 @@ def transform_property(
if not (49 <= lat <= 56 and -7 <= lng <= 2): if not (49 <= lat <= 56 and -7 <= lng <= 2):
return None return None
raw_beds = raw.get("beds") or 0 raw_beds = parse_int_value(raw.get("beds")) or 0
raw_baths = raw.get("baths") or 0 raw_baths = parse_int_value(raw.get("baths")) or 0
bedrooms = raw_beds if raw_beds <= MAX_BEDROOMS else 0 bedrooms = raw_beds if 0 <= raw_beds <= MAX_BEDROOMS else 0
bathrooms = raw_baths if raw_baths <= MAX_BEDROOMS else 0 bathrooms = raw_baths if 0 <= raw_baths <= MAX_BEDROOMS else 0
if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS: if raw_beds > MAX_BEDROOMS or raw_baths > MAX_BEDROOMS:
log.warning( log.warning(
"Zoopla %s: implausible beds=%d baths=%d (capped to 0)", "Zoopla %s: implausible beds=%d baths=%d (capped to 0)",
@ -683,9 +702,13 @@ def transform_property(
# Floor area: convert sq ft to sq m # Floor area: convert sq ft to sq m
floor_area_sqm = None floor_area_sqm = None
raw_sqm = raw.get("floor_area_sqm")
if raw_sqm:
floor_area_sqm = validate_floor_area(round(float(raw_sqm), 1))
else:
sqft = raw.get("floor_area_sqft") sqft = raw.get("floor_area_sqft")
if sqft: if sqft:
floor_area_sqm = validate_floor_area(round(sqft * 0.092903, 1)) floor_area_sqm = validate_floor_area(round(float(sqft) * 0.092903, 1))
listing_id = raw.get("id", "") listing_id = raw.get("id", "")
listing_url = raw.get("url", "") listing_url = raw.get("url", "")
@ -704,7 +727,7 @@ def transform_property(
"Leasehold/Freehold": raw.get("tenure") or None, "Leasehold/Freehold": raw.get("tenure") or None,
"Property type": _map_property_type(raw.get("property_type")), "Property type": _map_property_type(raw.get("property_type")),
"Property sub-type": normalize_sub_type(raw.get("property_type")), "Property sub-type": normalize_sub_type(raw.get("property_type")),
"price": int(price), "price": price,
"price_frequency": "", "price_frequency": "",
"Price qualifier": "", "Price qualifier": "",
"Total floor area (sqm)": floor_area_sqm, "Total floor area (sqm)": floor_area_sqm,
@ -760,7 +783,18 @@ def search_outcode(
properties = [] properties = []
dropped = 0 dropped = 0
for raw in raw_listings: for raw in raw_listings:
transformed = transform_property(raw, pc_index, pc_coords, search_outcode=outcode) try:
transformed = transform_property(
raw, pc_index, pc_coords, search_outcode=outcode
)
except Exception as exc:
log.warning(
"Zoopla %s property %s failed to transform: %s",
outcode,
raw.get("id", "?"),
exc,
)
transformed = None
if transformed: if transformed:
properties.append(transformed) properties.append(transformed)
else: else:

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

View file

@ -1,14 +1,16 @@
import { useMemo, useState } from 'react'; import { useMemo, useState, type MutableRefObject, type ReactNode } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server'; import { ts } from '../../i18n/server';
import type { import type {
FeatureFilters, FeatureFilters,
FeatureGroup,
FeatureMeta, FeatureMeta,
FilterExclusion, FilterExclusion,
HexagonStatsResponse, HexagonStatsResponse,
} from '../../types'; } from '../../types';
import { travelFieldKey, type TravelTimeEntry } from '../../hooks/useTravelTime'; import { travelFieldKey, type TravelTimeEntry } from '../../hooks/useTravelTime';
import type { HexagonLocation } from '../../lib/external-search'; import type { HexagonLocation } from '../../lib/external-search';
import { formatStationDistance, type NearbyStation } from '../../lib/nearby-stations';
import { import {
formatValue, formatValue,
formatFilterValue, formatFilterValue,
@ -16,19 +18,22 @@ import {
roundedPercentages, roundedPercentages,
} from '../../lib/format'; } from '../../lib/format';
import { groupFeaturesByCategory } from '../../lib/features'; import { groupFeaturesByCategory } from '../../lib/features';
import { getPoiCategoryLogoUrl } from '../../lib/map-utils';
import { import {
PARTY_FEATURE_COLORS, PARTY_FEATURE_COLORS,
STACKED_GROUPS, STACKED_GROUPS,
STACKED_ENUM_GROUPS, STACKED_ENUM_GROUPS,
STACKED_SEGMENT_COLORS, STACKED_SEGMENT_COLORS,
} from '../../lib/consts'; } from '../../lib/consts';
import { useNearbyStations } from '../../hooks/useNearbyStations';
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
import { DualHistogram, LoadingSkeleton } from './DualHistogram'; import { DualHistogram, LoadingSkeleton } from './DualHistogram';
import EnumBarChart from './EnumBarChart'; import EnumBarChart from './EnumBarChart';
import StackedBarChart from './StackedBarChart'; import StackedBarChart from './StackedBarChart';
import StackedEnumChart from './StackedEnumChart'; import StackedEnumChart from './StackedEnumChart';
import PriceHistoryChart from './PriceHistoryChart'; import PriceHistoryChart from './PriceHistoryChart';
import ExternalSearchLinks from './ExternalSearchLinks'; import ExternalSearchLinks from './ExternalSearchLinks';
import { InfoIcon } from '../ui/icons'; import { InfoIcon, TransitIcon } from '../ui/icons';
import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader'; import { CollapsibleGroupHeader } from '../ui/CollapsibleGroupHeader';
import { FeatureInfoPopup } from '../ui/FeatureInfoPopup'; import { FeatureInfoPopup } from '../ui/FeatureInfoPopup';
import { EmptyState } from '../ui/EmptyState'; import { EmptyState } from '../ui/EmptyState';
@ -54,6 +59,9 @@ interface AreaPaneProps {
shareCode?: string; shareCode?: string;
isGroupExpanded: (name: string) => boolean; isGroupExpanded: (name: string) => boolean;
onToggleGroup: (name: string) => void; onToggleGroup: (name: string) => void;
scrollTopRef?: MutableRefObject<number>;
scrollRestoreKey?: string | null;
scrollSaveDisabled?: boolean;
} }
function normalizePercentageSegments<T extends { value: number }>(segments: T[]): T[] { function normalizePercentageSegments<T extends { value: number }>(segments: T[]): T[] {
@ -75,6 +83,136 @@ function filterValueFormat(feature?: FeatureMeta) {
}; };
} }
const STATION_GROUP_NAME = 'Transport';
const STATION_GROUP_NAMES = new Set([STATION_GROUP_NAME, 'Public Transport']);
function MetricTextLabel({ children }: { children: ReactNode }) {
return (
<span className="block truncate text-[13px] font-medium leading-5 text-warm-900 dark:text-warm-100">
{children}
</span>
);
}
function MetricFeatureLabel({
feature,
onShowInfo,
label,
aboutLabel,
}: {
feature: FeatureMeta;
onShowInfo: (feature: FeatureMeta) => void;
label?: string;
aboutLabel: string;
}) {
return (
<div className="flex min-w-0 items-center gap-1.5">
<MetricTextLabel>{label ?? ts(feature.name)}</MetricTextLabel>
{feature.detail && (
<button
type="button"
onClick={() => onShowInfo(feature)}
className="-m-1 shrink-0 rounded p-1 text-warm-400 hover:bg-warm-100 hover:text-warm-700 dark:hover:bg-navy-800 dark:hover:text-warm-200"
title={aboutLabel}
aria-label={aboutLabel}
>
<InfoIcon className="h-3.5 w-3.5" />
</button>
)}
</div>
);
}
function MetricRow({
label,
chart,
value,
valueTitle,
className = '',
}: {
label: ReactNode;
chart?: ReactNode;
value?: ReactNode;
valueTitle?: string;
className?: string;
}) {
return (
<div
className={`grid min-h-10 grid-cols-[minmax(0,1fr)_6.5rem_minmax(3.5rem,auto)] items-center gap-3 py-1.5 ${className}`}
>
<div className="min-w-0">{label}</div>
<div className="w-[6.5rem] justify-self-end">{chart}</div>
<div
className="min-w-[3.5rem] max-w-[7rem] truncate text-right text-sm font-semibold leading-tight tabular-nums text-navy-950 dark:text-warm-50"
title={valueTitle}
>
{value}
</div>
</div>
);
}
function NearbyStationsCard({ location }: { location: HexagonLocation }) {
const { t } = useTranslation();
const origin = useMemo(
() => ({ lat: location.lat, lon: location.lon }),
[location.lat, location.lon]
);
const { stations, loading } = useNearbyStations(origin);
return (
<div className="py-1.5">
<div className="flex items-center gap-2 py-1">
<TransitIcon className="h-4 w-4 text-teal-600 dark:text-teal-400" />
<MetricTextLabel>{t('areaPane.closestStations')}</MetricTextLabel>
{loading && (
<span className="ml-auto h-3 w-3 rounded-full border-2 border-teal-600 border-t-transparent dark:border-teal-400 dark:border-t-transparent animate-spin" />
)}
</div>
{stations.length > 0 ? (
<ol className="divide-y divide-warm-100 dark:divide-navy-800">
{stations.map((station) => (
<NearbyStationRow key={station.id} station={station} />
))}
</ol>
) : (
<div className="py-2 text-sm text-warm-500 dark:text-warm-400">
{loading ? t('common.loading') : t('areaPane.noNearbyStations')}
</div>
)}
</div>
);
}
function NearbyStationRow({ station }: { station: NearbyStation }) {
const icon = getPoiCategoryLogoUrl(station.category, station.icon_category);
return (
<li className="flex items-center gap-2 px-3 py-2">
{icon ? (
<img
src={icon}
alt=""
aria-hidden="true"
loading="lazy"
className="h-5 w-5 shrink-0 rounded-[3px] bg-white object-contain p-0.5"
/>
) : (
<TransitIcon className="h-5 w-5 shrink-0 text-warm-400 dark:text-warm-500" />
)}
<div className="min-w-0 flex-1">
<div className="truncate text-sm font-medium text-warm-900 dark:text-warm-100">
{station.name}
</div>
<div className="text-xs text-warm-500 dark:text-warm-400">{ts(station.category)}</div>
</div>
<span className="shrink-0 text-sm font-semibold tabular-nums text-teal-700 dark:text-teal-400">
{formatStationDistance(station.distanceKm)}
</span>
</li>
);
}
export default function AreaPane({ export default function AreaPane({
stats, stats,
globalFeatures, globalFeatures,
@ -91,6 +229,9 @@ export default function AreaPane({
shareCode, shareCode,
isGroupExpanded, isGroupExpanded,
onToggleGroup, onToggleGroup,
scrollTopRef,
scrollRestoreKey,
scrollSaveDisabled,
}: AreaPaneProps) { }: AreaPaneProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const propertyCount = stats?.count; const propertyCount = stats?.count;
@ -99,7 +240,19 @@ export default function AreaPane({
const filteredStatsEmpty = filtersActive && statsUseFilters && stats?.count === 0; const filteredStatsEmpty = filtersActive && statsUseFilters && stats?.count === 0;
const showFlipToggleCallout = filteredStatsEmpty && unfilteredCount !== 0; const showFlipToggleCallout = filteredStatsEmpty && unfilteredCount !== 0;
const featureGroups = useMemo(() => groupFeaturesByCategory(globalFeatures), [globalFeatures]); const featureGroups = useMemo(() => groupFeaturesByCategory(globalFeatures), [globalFeatures]);
const displayFeatureGroups = useMemo<FeatureGroup[]>(() => {
if (!hexagonLocation || featureGroups.some((group) => STATION_GROUP_NAMES.has(group.name))) {
return featureGroups;
}
return [{ name: STATION_GROUP_NAME, features: [] }, ...featureGroups];
}, [featureGroups, hexagonLocation]);
const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null); const [infoFeature, setInfoFeature] = useState<FeatureMeta | null>(null);
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey: scrollRestoreKey ?? hexagonId,
scrollTopRef,
suspendSave: scrollSaveDisabled ?? (loading && stats == null),
});
const numericByName = useMemo(() => { const numericByName = useMemo(() => {
if (!stats) return new Map(); if (!stats) return new Map();
@ -164,7 +317,7 @@ export default function AreaPane({
<> <>
<div className="relative flex h-full flex-col"> <div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && stats != null} /> <IndeterminateProgressBar show={loading && stats != null} />
<div className="flex-1 overflow-y-auto"> <div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto">
<div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950"> <div className="border-b border-warm-200 bg-white dark:border-navy-700 dark:bg-navy-950">
<div className="space-y-3 p-3"> <div className="space-y-3 p-3">
<div className="flex items-start justify-between gap-3"> <div className="flex items-start justify-between gap-3">
@ -300,20 +453,22 @@ export default function AreaPane({
{stats.price_history && {stats.price_history &&
(() => { (() => {
const uniqueYears = new Set(stats.price_history.map((p) => Math.floor(p.year))); const uniqueYears = new Set(stats.price_history.map((p) => Math.floor(p.year)));
return uniqueYears.size > 1; return uniqueYears.size > 1 ? (
})() && (
<div className="mx-3 mt-2 bg-warm-50 dark:bg-warm-800 rounded p-2"> <div className="mx-3 mt-2 bg-warm-50 dark:bg-warm-800 rounded p-2">
<span className="text-xs text-warm-700 dark:text-warm-300"> <span className="text-xs text-warm-700 dark:text-warm-300">
{t('areaPane.priceHistory')} {t('areaPane.priceHistory')}
</span> </span>
<PriceHistoryChart points={stats.price_history} /> <PriceHistoryChart points={stats.price_history} />
</div> </div>
)} ) : null;
{featureGroups.map((group) => { })()}
{displayFeatureGroups.map((group) => {
const showNearbyStations =
hexagonLocation != null && STATION_GROUP_NAMES.has(group.name);
const hasData = group.features.some( const hasData = group.features.some(
(feature) => numericByName.has(feature.name) || enumByName.has(feature.name) (feature) => numericByName.has(feature.name) || enumByName.has(feature.name)
); );
if (!hasData) return null; if (!hasData && !showNearbyStations) return null;
const stackedCharts = STACKED_GROUPS[group.name]; const stackedCharts = STACKED_GROUPS[group.name];
const stackedEnumCharts = STACKED_ENUM_GROUPS[group.name]; const stackedEnumCharts = STACKED_ENUM_GROUPS[group.name];
@ -332,10 +487,11 @@ export default function AreaPane({
name={group.name} name={group.name}
expanded={expanded} expanded={expanded}
onToggle={() => onToggleGroup(group.name)} onToggle={() => onToggleGroup(group.name)}
className="px-3 py-2.5 text-sm font-bold text-warm-500 bg-warm-50 dark:bg-warm-900 dark:text-warm-400 sticky top-0 z-10 hover:bg-warm-100 dark:hover:bg-warm-800" className="area-pane-group-header sticky top-0 z-10 bg-white px-3 pb-1.5 pt-4 text-[11px] font-bold uppercase tracking-wide text-warm-500 hover:bg-warm-50 dark:bg-navy-950 dark:text-warm-400 dark:hover:bg-navy-900"
/> />
{expanded && ( {expanded && (
<div className="px-3 py-2 space-y-3"> <div className="divide-y divide-warm-100 px-3 py-1 dark:divide-navy-800">
{showNearbyStations && <NearbyStationsCard location={hexagonLocation} />}
{stackedCharts?.map((chart) => { {stackedCharts?.map((chart) => {
const segments = chart.components const segments = chart.components
.map((name) => ({ .map((name) => ({
@ -445,21 +601,17 @@ export default function AreaPane({
: undefined; : undefined;
return ( return (
<div <MetricRow
key={feature.name} key={feature.name}
className="bg-warm-50 dark:bg-warm-800 rounded p-2" label={
> <MetricFeatureLabel
<div className="flex justify-between items-baseline">
<FeatureLabel
feature={feature} feature={feature}
onShowInfo={setInfoFeature} onShowInfo={setInfoFeature}
className="mr-2" aboutLabel={t('filters.aboutData')}
/> />
<span className="text-xs font-semibold text-teal-700 dark:text-teal-400 whitespace-nowrap"> }
{formatValue(numericStats.mean, feature)} chart={
</span> numericStats.histogram &&
</div>
{numericStats.histogram &&
(globalHistogram ? ( (globalHistogram ? (
<DualHistogram <DualHistogram
localCounts={numericStats.histogram.counts} localCounts={numericStats.histogram.counts}
@ -476,6 +628,8 @@ export default function AreaPane({
: feature.raw : feature.raw
) )
} }
integerAxisLabels={feature.step === 1}
compact
/> />
) : ( ) : (
<DualHistogram <DualHistogram
@ -491,9 +645,18 @@ export default function AreaPane({
: feature.raw : feature.raw
) )
} }
integerAxisLabels={feature.step === 1}
compact
/>
))
}
value={formatValue(numericStats.mean, feature)}
valueTitle={
globalMean != null
? `${t('areaPane.nationalAvg')}: ${formatValue(globalMean)}`
: undefined
}
/> />
))}
</div>
); );
} }

View file

@ -0,0 +1,26 @@
import { describe, expect, it } from 'vitest';
import { compactHistogramLabel } from './DualHistogram';
describe('compactHistogramLabel', () => {
it('rounds low-cardinality count labels to integers', () => {
const fmt = (value: number) => value.toFixed(2);
const labels = [0, 0.99, 2.98, 4.96, 5.95].map((center, index) =>
compactHistogramLabel(index, 5, 0, 5.95, center, fmt, true)
);
expect(labels).toEqual(['0', '1', '3', '5', '6+']);
});
it('labels the first integer count bucket as zero when it means below one', () => {
const fmt = (value: number) => value.toFixed(2);
expect(compactHistogramLabel(0, 5, 0.99, 5.95, 0.99, fmt, true)).toBe('0');
});
it('keeps fractional labels when integer labels are not requested', () => {
const fmt = (value: number) => value.toFixed(2);
expect(compactHistogramLabel(1, 5, 0, 5.95, 0.99, fmt, false)).toBe('0.99');
});
});

View file

@ -30,6 +30,42 @@ function pickTicks(min: number, max: number, count: number): number[] {
return ticks; return ticks;
} }
function isLowCardinalityHistogram(counts: number[], p1: number, p99: number): boolean {
return counts.length > 0 && counts.length <= 10 && p99 > p1 && p99 - p1 <= 10;
}
export function compactHistogramLabel(
index: number,
barCount: number,
p1: number,
p99: number,
center: number,
formatLabel: (value: number) => string,
integerLabels = false
): string {
const formatAxisValue = (value: number) =>
integerLabels ? Math.round(value).toLocaleString() : formatLabel(value);
if (barCount <= 1) return formatAxisValue(center);
const middleBins = barCount - 2;
if (index === 0) {
if (!integerLabels) return `<${formatLabel(p1)}`;
const firstBoundary = Math.ceil(p1);
return firstBoundary <= 1 ? '0' : `<${firstBoundary.toLocaleString()}`;
}
if (index === barCount - 1) {
if (!integerLabels) return `${formatLabel(p99)}+`;
return `${Math.ceil(p99).toLocaleString()}+`;
}
const middleWidth = middleBins > 0 ? (p99 - p1) / middleBins : 0;
if (Math.abs(middleWidth - 1) < 0.001) {
return formatAxisValue(p1 + index - 1);
}
return formatAxisValue(center);
}
export function DualHistogram({ export function DualHistogram({
localCounts, localCounts,
globalCounts, globalCounts,
@ -38,6 +74,8 @@ export function DualHistogram({
globalMean, globalMean,
meanLabel, meanLabel,
formatLabel, formatLabel,
compact = false,
integerAxisLabels = false,
}: { }: {
localCounts: number[]; localCounts: number[];
globalCounts: number[]; globalCounts: number[];
@ -46,9 +84,15 @@ export function DualHistogram({
globalMean?: number; globalMean?: number;
meanLabel?: string; meanLabel?: string;
formatLabel?: (value: number) => string; formatLabel?: (value: number) => string;
compact?: boolean;
integerAxisLabels?: boolean;
}) { }) {
const { t } = useTranslation(); const { t } = useTranslation();
const targetBars = 25; const showCompactAxisLabels =
compact &&
isLowCardinalityHistogram(localCounts, p1, p99) &&
isLowCardinalityHistogram(globalCounts, p1, p99);
const targetBars = compact ? (showCompactAxisLabels ? localCounts.length : 16) : 25;
const localBars = downsampleBars(localCounts, targetBars); const localBars = downsampleBars(localCounts, targetBars);
const globalBars = downsampleBars(globalCounts, targetBars); const globalBars = downsampleBars(globalCounts, targetBars);
@ -59,6 +103,8 @@ export function DualHistogram({
const fmt = const fmt =
formatLabel ?? ((v: number) => (Number.isInteger(v) ? v.toLocaleString() : v.toFixed(1))); formatLabel ?? ((v: number) => (Number.isInteger(v) ? v.toLocaleString() : v.toFixed(1)));
if (barCount === 0) return null;
// Compute center value for each bar. // Compute center value for each bar.
// Bar 0 = low outlier, bars 1..n-2 = middle (p1 to p99), bar n-1 = high outlier. // Bar 0 = low outlier, bars 1..n-2 = middle (p1 to p99), bar n-1 = high outlier.
const middleBins = Math.max(barCount - 2, 0); const middleBins = Math.max(barCount - 2, 0);
@ -97,6 +143,60 @@ export function DualHistogram({
? { right: 0 } ? { right: 0 }
: { left: '50%', transform: 'translateX(-50%)' }; : { left: '50%', transform: 'translateX(-50%)' };
if (compact) {
const axisLabels = showCompactAxisLabels
? barCenters.map((center, index) =>
compactHistogramLabel(index, barCount, p1, p99, center, fmt, integerAxisLabels)
)
: [];
const chartTitle = [
`${fmt(p1)} - ${fmt(p99)}`,
globalMean != null ? `${meanLabel ?? t('areaPane.nationalAvg')}: ${fmt(globalMean)}` : null,
]
.filter(Boolean)
.join('\n');
return (
<div className={showCompactAxisLabels ? 'h-10' : 'h-7'} title={chartTitle}>
<div
className={`${showCompactAxisLabels ? 'h-7' : 'h-full'} relative flex items-end gap-[2px]`}
>
{Array.from({ length: barCount }).map((_, index) => {
const globalHeight = (globalBars[index] / globalMax) * 100;
const localHeight = (localBars[index] / localMax) * 100;
return (
<div key={index} className="relative flex h-full min-w-[2px] flex-1 items-end">
<div
className="absolute bottom-0 left-0 right-0 rounded-t-[2px] bg-warm-300/45 dark:bg-warm-600/50"
style={{ height: `${Math.max(globalHeight, globalBars[index] > 0 ? 8 : 0)}%` }}
/>
{localBars[index] > 0 && (
<div
className="absolute bottom-0 left-[22%] right-[22%] rounded-t-[2px] bg-teal-600 dark:bg-teal-400"
style={{ height: `${Math.max(localHeight, 12)}%` }}
/>
)}
</div>
);
})}
</div>
{showCompactAxisLabels && (
<div className="mt-0.5 flex gap-[2px]">
{axisLabels.map((label, index) => (
<span
key={`${label}-${index}`}
className="min-w-[2px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={label}
>
{label}
</span>
))}
</div>
)}
</div>
);
}
return ( return (
<div className="mt-1"> <div className="mt-1">
<div className={showMeanMarker ? 'relative pt-5' : 'relative'}> <div className={showMeanMarker ? 'relative pt-5' : 'relative'}>
@ -152,35 +252,29 @@ export function DualHistogram({
function SkeletonHistogram() { function SkeletonHistogram() {
return ( return (
<div className="bg-warm-50 dark:bg-warm-800 rounded p-2 animate-pulse"> <div className="grid min-h-10 animate-pulse grid-cols-[minmax(0,1fr)_6.5rem_minmax(3.5rem,auto)] items-center gap-3 py-1.5">
<div className="flex justify-between items-baseline"> <div className="h-3 w-24 rounded bg-warm-200 dark:bg-warm-700" />
<div className="h-3 w-24 bg-warm-200 dark:bg-warm-700 rounded" /> <div className="flex h-7 items-end gap-[2px]">
<div className="h-3 w-10 bg-warm-200 dark:bg-warm-700 rounded" /> {Array.from({ length: 12 }).map((_, i) => (
</div>
<div className="flex items-end gap-px h-10 mt-2">
{Array.from({ length: 15 }).map((_, i) => (
<div <div
key={i} key={i}
className="flex-1 bg-warm-200 dark:bg-warm-700 rounded-t-sm min-w-[2px]" className="min-w-[2px] flex-1 rounded-t-[2px] bg-warm-200 dark:bg-warm-700"
style={{ height: `${20 + Math.sin(i * 0.7) * 30 + 30}%` }} style={{ height: `${22 + Math.sin(i * 0.7) * 28 + 30}%` }}
/> />
))} ))}
</div> </div>
<div className="flex justify-between mt-1"> <div className="h-3 w-10 justify-self-end rounded bg-warm-200 dark:bg-warm-700" />
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
<div className="h-2 w-6 bg-warm-200 dark:bg-warm-700 rounded" />
</div>
</div> </div>
); );
} }
export function LoadingSkeleton() { export function LoadingSkeleton() {
return ( return (
<div className="p-3 space-y-4"> <div className="space-y-4 p-3">
{[0, 1, 2].map((groupIdx) => ( {[0, 1, 2].map((groupIdx) => (
<div key={groupIdx}> <div key={groupIdx}>
<div className="h-3 w-20 bg-warm-200 dark:bg-warm-700 rounded animate-pulse mb-2" /> <div className="mb-2 h-3 w-20 animate-pulse rounded bg-warm-200 dark:bg-warm-700" />
<div className="space-y-3"> <div className="divide-y divide-warm-100 dark:divide-navy-800">
{Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => ( {Array.from({ length: groupIdx === 0 ? 3 : 2 }).map((_, i) => (
<SkeletonHistogram key={i} /> <SkeletonHistogram key={i} />
))} ))}

View file

@ -1,16 +1,34 @@
import { ts } from '../../i18n/server'; import { ts } from '../../i18n/server';
import { getEnumValueColor } from '../../lib/consts'; import { getEnumValueColor } from '../../lib/consts';
function shortenAxisLabel(label: string, total: number): string {
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label)
.slice(0, total <= 5 ? 3 : 2)
.join('');
}
export default function EnumBarChart({ export default function EnumBarChart({
counts, counts,
globalCounts, globalCounts,
featureName, featureName,
compact = false,
}: { }: {
counts: Record<string, number>; counts: Record<string, number>;
globalCounts?: Record<string, number>; globalCounts?: Record<string, number>;
featureName: string; featureName: string;
compact?: boolean;
}) { }) {
const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA); const entries = Object.entries(counts).sort(([, countA], [, countB]) => countB - countA);
if (entries.length === 0) return null;
const localTotal = entries.reduce((sum, [, c]) => sum + c, 0); const localTotal = entries.reduce((sum, [, c]) => sum + c, 0);
// When global counts are available, normalize both to percentages for comparison // When global counts are available, normalize both to percentages for comparison
@ -28,6 +46,71 @@ export default function EnumBarChart({
// Fallback to raw count scaling when no global data // Fallback to raw count scaling when no global data
const maxCount = Math.max(...entries.map(([, count]) => count), 1); const maxCount = Math.max(...entries.map(([, count]) => count), 1);
if (compact) {
const title = entries
.map(([label, count]) => {
const localPct = localTotal > 0 ? (count / localTotal) * 100 : 0;
const globalPct =
hasGlobal && globalTotal > 0 ? ((globalCounts[label] ?? 0) / globalTotal) * 100 : null;
return `${ts(label)}: ${count.toLocaleString()} (${localPct.toFixed(1)}%)${
globalPct != null ? ` / ${globalPct.toFixed(1)}%` : ''
}`;
})
.join('\n');
return (
<div className="h-10" title={title}>
<div className="flex h-7 items-end gap-[2px]">
{entries.map(([label, count]) => {
const localPct = localTotal > 0 ? count / localTotal : 0;
const globalPct = hasGlobal ? (globalCounts[label] ?? 0) / globalTotal : 0;
const localHeight = hasGlobal
? maxPct > 0
? (localPct / maxPct) * 100
: 0
: (count / maxCount) * 100;
const globalHeight = hasGlobal && maxPct > 0 ? (globalPct / maxPct) * 100 : 0;
const color = getEnumValueColor(featureName, label);
return (
<div key={label} className="relative flex h-full min-w-[3px] flex-1 items-end">
{hasGlobal && (
<div
className="absolute bottom-0 left-0 right-0 rounded-t-[2px] bg-warm-300/45 dark:bg-warm-600/50"
style={{ height: `${Math.max(globalHeight, globalPct > 0 ? 8 : 0)}%` }}
/>
)}
{count > 0 && (
<div
className="absolute bottom-0 left-[18%] right-[18%] rounded-t-[2px]"
style={{
height: `${Math.max(localHeight, 12)}%`,
backgroundColor: `rgb(${color[0]},${color[1]},${color[2]})`,
}}
/>
)}
</div>
);
})}
</div>
<div className="mt-0.5 flex gap-[2px]">
{entries.map(([label]) => {
const translated = ts(label);
return (
<span
key={label}
className="min-w-[3px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={translated}
>
{shortenAxisLabel(translated, entries.length)}
</span>
);
})}
</div>
</div>
);
}
return ( return (
<div className="space-y-1 mt-1"> <div className="space-y-1 mt-1">
{entries.map(([label, count]) => { {entries.map(([label, count]) => {

View file

@ -3,36 +3,19 @@ import { useTranslation } from 'react-i18next';
export default function HistogramLegend() { export default function HistogramLegend() {
const { t } = useTranslation(); const { t } = useTranslation();
return ( return (
<div className="mx-3 mt-3 bg-teal-50 dark:bg-teal-900/20 border border-teal-200 dark:border-teal-800/50 rounded p-2.5 text-xs"> <div className="mx-3 mt-3 flex flex-wrap items-center gap-x-3 gap-y-1 rounded border border-warm-200 bg-white px-2.5 py-1.5 text-[10px] text-warm-500 dark:border-navy-800 dark:bg-navy-950/60 dark:text-warm-400">
<div className="space-y-1.5"> <div className="flex items-center gap-1.5">
<div className="flex items-center gap-2"> <div className="h-2.5 w-2 rounded-[2px] bg-teal-600 dark:bg-teal-400" />
<div className="w-3 h-3 bg-teal-500 dark:bg-teal-400 rounded" /> <span className="font-medium text-warm-700 dark:text-warm-200">
<span className="text-warm-700 dark:text-warm-300">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.tealBars')} {t('histogramLegend.tealBars')}
</span>{' '}
{t('histogramLegend.tealBarsDesc')}
</span> </span>
</div> </div>
<div className="flex items-center gap-2"> <div className="flex items-center gap-1.5">
<div className="w-3 h-3 bg-warm-300/60 dark:bg-warm-600/60 rounded" /> <div className="h-2.5 w-2 rounded-[2px] bg-warm-300/70 dark:bg-warm-600/70" />
<span className="text-warm-700 dark:text-warm-300"> <span className="font-medium text-warm-700 dark:text-warm-200">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.greyBars')} {t('histogramLegend.greyBars')}
</span>{' '}
{t('histogramLegend.greyBarsDesc')}
</span> </span>
</div> </div>
<div className="flex items-center gap-2">
<div className="w-3 h-px border-t border-dashed border-warm-500 dark:border-warm-400" />
<span className="text-warm-700 dark:text-warm-300">
<span className="font-medium text-warm-900 dark:text-warm-100">
{t('histogramLegend.dashedLine')}
</span>{' '}
{t('histogramLegend.dashedLineDesc')}
</span>
</div>
</div>
</div> </div>
); );
} }

View file

@ -147,6 +147,8 @@ export default function MapPage({
const pendingCurrentLocationFlyToRef = useRef<{ lat: number; lng: number } | null>(null); const pendingCurrentLocationFlyToRef = useRef<{ lat: number; lng: number } | null>(null);
const pendingLocationSearchFlyToRef = useRef<PendingFlyTo | null>(null); const pendingLocationSearchFlyToRef = useRef<PendingFlyTo | null>(null);
const mobileDrawerPanelRectRef = useRef<DOMRectReadOnly | null>(null); const mobileDrawerPanelRectRef = useRef<DOMRectReadOnly | null>(null);
const areaPaneScrollTopRef = useRef(0);
const propertiesPaneScrollTopRef = useRef(0);
const getMobileMapFlyToOptions = useCallback((): MapFlyToOptions | undefined => { const getMobileMapFlyToOptions = useCallback((): MapFlyToOptions | undefined => {
if (!isMobile) return undefined; if (!isMobile) return undefined;
@ -558,6 +560,11 @@ export default function MapPage({
shareCode={shareCode} shareCode={shareCode}
isGroupExpanded={isAreaGroupExpanded} isGroupExpanded={isAreaGroupExpanded}
onToggleGroup={toggleAreaGroup} onToggleGroup={toggleAreaGroup}
scrollTopRef={areaPaneScrollTopRef}
scrollRestoreKey={
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
}
scrollSaveDisabled={loadingAreaStats && areaStats == null}
/> />
</Suspense> </Suspense>
); );
@ -570,6 +577,11 @@ export default function MapPage({
loading={loadingProperties} loading={loadingProperties}
hexagonId={selectedHexagon?.id || null} hexagonId={selectedHexagon?.id || null}
onLoadMore={handleLoadMoreProperties} onLoadMore={handleLoadMoreProperties}
scrollTopRef={propertiesPaneScrollTopRef}
scrollRestoreKey={
selectedHexagon ? `${selectedHexagon.type}:${selectedHexagon.id}` : null
}
scrollSaveDisabled={loadingProperties && properties.length === 0}
/> />
</Suspense> </Suspense>
); );

View file

@ -3,7 +3,7 @@ import { useTranslation } from 'react-i18next';
import { ts } from '../../i18n/server'; import { ts } from '../../i18n/server';
import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups'; import { useCollapsibleGroups } from '../../hooks/useCollapsibleGroups';
import { trackEvent } from '../../lib/analytics'; import { trackEvent } from '../../lib/analytics';
import { POI_CATEGORY_LOGOS } from '../../lib/consts'; import { getPoiCategoryLogoUrl } from '../../lib/map-utils';
import type { POICategoryGroup } from '../../types'; import type { POICategoryGroup } from '../../types';
import InfoPopup from '../ui/InfoPopup'; import InfoPopup from '../ui/InfoPopup';
import { SearchInput } from '../ui/SearchInput'; import { SearchInput } from '../ui/SearchInput';
@ -188,7 +188,7 @@ export default function POIPane({
<div className="px-3 py-2"> <div className="px-3 py-2">
<PillGroup> <PillGroup>
{group.categories.map((category) => { {group.categories.map((category) => {
const logo = POI_CATEGORY_LOGOS[category]; const logo = getPoiCategoryLogoUrl(category);
return ( return (
<PillToggle <PillToggle
key={category} key={category}

View file

@ -1,8 +1,9 @@
import { useMemo, useState, useEffect } from 'react'; import { useMemo, useState, useEffect, type MutableRefObject } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { Property } from '../../types'; import { Property } from '../../types';
import { formatDuration, formatAge, formatNumber, formatTransactionDate } from '../../lib/format'; import { formatDuration, formatAge, formatNumber, formatTransactionDate } from '../../lib/format';
import { getNum } from '../../lib/property-fields'; import { getNum } from '../../lib/property-fields';
import { useRetainedScrollTop } from '../../hooks/useRetainedScrollTop';
import InfoPopup from '../ui/InfoPopup'; import InfoPopup from '../ui/InfoPopup';
import { SearchInput } from '../ui/SearchInput'; import { SearchInput } from '../ui/SearchInput';
import { EmptyState } from '../ui/EmptyState'; import { EmptyState } from '../ui/EmptyState';
@ -17,6 +18,9 @@ interface PropertiesPaneProps {
hexagonId: string | null; hexagonId: string | null;
onLoadMore: () => void; onLoadMore: () => void;
onNavigateToSource?: (slug: string) => void; onNavigateToSource?: (slug: string) => void;
scrollTopRef?: MutableRefObject<number>;
scrollRestoreKey?: string | null;
scrollSaveDisabled?: boolean;
} }
export function PropertiesPane({ export function PropertiesPane({
@ -26,10 +30,18 @@ export function PropertiesPane({
hexagonId, hexagonId,
onLoadMore, onLoadMore,
onNavigateToSource, onNavigateToSource,
scrollTopRef,
scrollRestoreKey,
scrollSaveDisabled,
}: PropertiesPaneProps) { }: PropertiesPaneProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const [search, setSearch] = useState(''); const [search, setSearch] = useState('');
const [showInfo, setShowInfo] = useState(false); const [showInfo, setShowInfo] = useState(false);
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey: scrollRestoreKey ?? hexagonId,
scrollTopRef,
suspendSave: scrollSaveDisabled ?? (loading && properties.length === 0),
});
useEffect(() => { useEffect(() => {
setSearch(''); setSearch('');
@ -60,7 +72,7 @@ export function PropertiesPane({
return ( return (
<div className="relative flex h-full flex-col"> <div className="relative flex h-full flex-col">
<IndeterminateProgressBar show={loading && properties.length > 0} /> <IndeterminateProgressBar show={loading && properties.length > 0} />
<div className="flex-1 overflow-y-auto"> <div ref={scrollRef} onScroll={onScroll} className="flex-1 overflow-y-auto">
{showInfo && ( {showInfo && (
<InfoPopup <InfoPopup
title={t('propertyCard.propertyData')} title={t('propertyCard.propertyData')}

View file

@ -12,6 +12,7 @@ interface StackedBarChartProps {
segments: Segment[]; segments: Segment[];
total: number; total: number;
colorMap: Record<string, string>; colorMap: Record<string, string>;
compact?: boolean;
} }
/** Strip common suffixes/prefixes to produce short legend labels */ /** Strip common suffixes/prefixes to produce short legend labels */
@ -28,7 +29,27 @@ function shortenLabel(name: string): string {
.trim(); .trim();
} }
export default function StackedBarChart({ segments, total, colorMap }: StackedBarChartProps) { function shortenAxisLabel(name: string, total: number): string {
const label = shortenLabel(name);
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label)
.slice(0, total <= 5 ? 3 : 2)
.join('');
}
export default function StackedBarChart({
segments,
total,
colorMap,
compact = false,
}: StackedBarChartProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const sortedSegments = useMemo(() => [...segments].sort((a, b) => b.value - a.value), [segments]); const sortedSegments = useMemo(() => [...segments].sort((a, b) => b.value - a.value), [segments]);
const roundedPcts = useMemo( const roundedPcts = useMemo(
@ -55,6 +76,53 @@ export default function StackedBarChart({ segments, total, colorMap }: StackedBa
return color; return color;
}; };
if (compact) {
const maxValue = Math.max(...sortedSegments.map((segment) => segment.value), 1);
const showAxisLabels = sortedSegments.length <= 8;
const title = sortedSegments
.map((segment, i) => {
const label = shortenLabel(ts(segment.name));
return `${label}: ${formatValue(segment.value)} (${roundedPcts[i].toFixed(1)}%)`;
})
.join('\n');
return (
<div className={showAxisLabels ? 'h-10' : 'h-7'} title={title}>
<div className={`${showAxisLabels ? 'h-7' : 'h-full'} flex items-end gap-[2px]`}>
{sortedSegments.map((segment) => {
const height = (segment.value / maxValue) * 100;
return (
<div
key={segment.name}
className="min-w-[3px] flex-1 rounded-t-[2px]"
style={{
height: `${Math.max(height, 12)}%`,
backgroundColor: colorFor(segment.name),
}}
/>
);
})}
</div>
{showAxisLabels && (
<div className="mt-0.5 flex gap-[2px]">
{sortedSegments.map((segment) => {
const label = shortenLabel(ts(segment.name));
return (
<span
key={segment.name}
className="min-w-[3px] flex-1 truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={label}
>
{shortenAxisLabel(label, sortedSegments.length)}
</span>
);
})}
</div>
)}
</div>
);
}
return ( return (
<div className="space-y-1.5"> <div className="space-y-1.5">
{/* Stacked bar */} {/* Stacked bar */}

View file

@ -7,6 +7,7 @@ interface StackedEnumChartProps {
components: { label: string; stats: EnumFeatureStats }[]; components: { label: string; stats: EnumFeatureStats }[];
valueOrder: string[]; valueOrder: string[];
valueColors: string[]; valueColors: string[];
compact?: boolean;
} }
/** Strip common suffixes to produce short row labels */ /** Strip common suffixes to produce short row labels */
@ -14,10 +15,24 @@ function shortenLabel(name: string): string {
return name.replace(/ risk$/, ''); return name.replace(/ risk$/, '');
} }
function shortenAxisLabel(name: string): string {
const label = shortenLabel(name);
if (label.length <= 3) return label;
const parts = label.split(/[\s/&-]+/).filter(Boolean);
if (parts.length > 1) {
return parts
.map((part) => Array.from(part)[0])
.join('')
.slice(0, 3);
}
return Array.from(label).slice(0, 3).join('');
}
export default function StackedEnumChart({ export default function StackedEnumChart({
components, components,
valueOrder, valueOrder,
valueColors, valueColors,
compact = false,
}: StackedEnumChartProps) { }: StackedEnumChartProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const visibleRows = components.filter(({ stats }) => { const visibleRows = components.filter(({ stats }) => {
@ -35,6 +50,63 @@ export default function StackedEnumChart({
); );
} }
if (compact) {
return (
<div className="divide-y divide-warm-100 dark:divide-navy-800">
{visibleRows.map(({ label, stats }) => {
const counts = valueOrder.map((value) => stats.counts[value] ?? 0);
const total = counts.reduce((a, b) => a + b, 0);
const roundedPcts = roundedPercentages(counts, total, 0);
const title = valueOrder
.map((value, i) => `${ts(value)}: ${counts[i]} (${roundedPcts[i]}%)`)
.join('\n');
return (
<div
key={label}
className="grid min-h-8 grid-cols-[minmax(0,1fr)_6.5rem] items-center gap-3 py-1.5"
>
<span className="truncate text-xs font-medium text-warm-800 dark:text-warm-200">
{shortenLabel(ts(label))}
</span>
<div
className="flex h-5 overflow-hidden rounded-sm bg-warm-200 dark:bg-warm-700"
title={title}
>
{valueOrder.map((value, i) => {
const count = counts[i];
const pct = (count / total) * 100;
if (pct < 0.5) return null;
return (
<div
key={value}
className="h-full"
style={{
width: `${pct}%`,
backgroundColor: valueColors[i],
}}
/>
);
})}
</div>
</div>
);
})}
<div className="ml-auto grid w-[6.5rem] grid-flow-col auto-cols-fr gap-1 pt-1">
{valueOrder.map((value) => (
<span
key={value}
className="truncate text-center text-[8px] font-medium leading-none text-warm-500 dark:text-warm-400"
title={ts(value)}
>
{shortenAxisLabel(ts(value))}
</span>
))}
</div>
</div>
);
}
return ( return (
<div className="space-y-1.5"> <div className="space-y-1.5">
{visibleRows.map(({ label, stats }) => { {visibleRows.map(({ label, stats }) => {

View file

@ -110,7 +110,7 @@ export function ActiveFiltersPanel({
> >
<button <button
onClick={onToggleCollapsed} onClick={onToggleCollapsed}
className="shrink-0 flex items-center justify-between border-y border-l-4 border-teal-300 border-l-teal-600 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:border-l-teal-300 dark:bg-teal-900/50 dark:hover:bg-teal-900/70" className="shrink-0 flex items-center justify-between border-y border-teal-300 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
> >
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<span className="text-sm font-bold text-navy-950 dark:text-warm-100"> <span className="text-sm font-bold text-navy-950 dark:text-warm-100">

View file

@ -110,7 +110,7 @@ export function AddFilterPanel({
> >
<button <button
onClick={onToggleCollapsed} onClick={onToggleCollapsed}
className="shrink-0 flex items-center justify-between border-y border-l-4 border-teal-300 border-l-teal-600 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:border-l-teal-300 dark:bg-teal-900/50 dark:hover:bg-teal-900/70" className="shrink-0 flex items-center justify-between border-y border-teal-300 bg-teal-100 px-3 py-3 cursor-pointer hover:bg-teal-200 dark:border-teal-800 dark:bg-teal-900/50 dark:hover:bg-teal-900/70"
> >
<span className="text-sm font-bold text-navy-950 dark:text-warm-100"> <span className="text-sm font-bold text-navy-950 dark:text-warm-100">
{t('filters.addFilter')} {t('filters.addFilter')}
@ -122,8 +122,8 @@ export function AddFilterPanel({
</button> </button>
{(!collapsed || !isLicensed) && ( {(!collapsed || !isLicensed) && (
<div className="flex min-h-0 flex-1 flex-col"> <div className="flex min-h-0 flex-1 flex-col">
<div className="flex min-h-0 flex-1 flex-col overflow-y-auto">
{!collapsed && ( {!collapsed && (
<div className="min-h-0 flex-1 overflow-y-auto">
<FeatureBrowser <FeatureBrowser
availableFeatures={availableFeatures} availableFeatures={availableFeatures}
allFeatures={allFeatures} allFeatures={allFeatures}
@ -136,6 +136,7 @@ export function AddFilterPanel({
travelTimeEntries={travelTimeEntries} travelTimeEntries={travelTimeEntries}
onAddTravelTimeEntry={onAddTravelTimeEntry} onAddTravelTimeEntry={onAddTravelTimeEntry}
/> />
</div>
)} )}
{!isLicensed && ( {!isLicensed && (
<div className="mt-auto shrink-0 flex flex-col items-center px-5 pt-4 pb-0 border-t border-warm-200 dark:border-warm-700"> <div className="mt-auto shrink-0 flex flex-col items-center px-5 pt-4 pb-0 border-t border-warm-200 dark:border-warm-700">
@ -167,7 +168,6 @@ export function AddFilterPanel({
</div> </div>
)} )}
</div> </div>
</div>
)} )}
</div> </div>
); );

View file

@ -1,5 +1,7 @@
import { useRef, useCallback, useEffect, useId, type ReactNode } from 'react'; import { useCallback, useEffect, useId, type ReactNode } from 'react';
import { createPortal } from 'react-dom';
import { useClickOutside } from '../../hooks/useClickOutside'; import { useClickOutside } from '../../hooks/useClickOutside';
import { useModalA11y } from '../../hooks/useModalA11y';
import { CloseIcon } from './icons'; import { CloseIcon } from './icons';
import { IconButton } from './IconButton'; import { IconButton } from './IconButton';
@ -11,8 +13,7 @@ interface InfoPopupProps {
} }
export default function InfoPopup({ title, children, onClose, sourceLink }: InfoPopupProps) { export default function InfoPopup({ title, children, onClose, sourceLink }: InfoPopupProps) {
const popupRef = useRef<HTMLDivElement>(null); const popupRef = useModalA11y();
const previouslyFocusedRef = useRef<HTMLElement | null>(null);
const titleId = useId(); const titleId = useId();
const handleClose = useCallback(() => { const handleClose = useCallback(() => {
@ -29,20 +30,9 @@ export default function InfoPopup({ title, children, onClose, sourceLink }: Info
return () => document.removeEventListener('keydown', handleKeyDown); return () => document.removeEventListener('keydown', handleKeyDown);
}, [onClose]); }, [onClose]);
useEffect(() => { const popup = (
previouslyFocusedRef.current = document.activeElement as HTMLElement | null;
const firstFocusable = popupRef.current?.querySelector<HTMLElement>(
'button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])'
);
(firstFocusable ?? popupRef.current)?.focus();
return () => {
previouslyFocusedRef.current?.focus?.();
};
}, []);
return (
<div <div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/30 p-4" className="fixed inset-0 z-[10000] flex items-center justify-center bg-black/50 p-4 dark:bg-black/70"
role="presentation" role="presentation"
> >
<div <div
@ -73,4 +63,8 @@ export default function InfoPopup({ title, children, onClose, sourceLink }: Info
</div> </div>
</div> </div>
); );
if (typeof document === 'undefined') return popup;
return createPortal(popup, document.body);
} }

View file

@ -160,7 +160,7 @@ export default function MobileMenu({
<div className="fixed inset-0 bg-black/50 z-[70]" onClick={onClose} /> <div className="fixed inset-0 bg-black/50 z-[70]" onClick={onClose} />
{/* Menu panel */} {/* Menu panel */}
<div className="mobile-menu-panel fixed top-0 right-0 bottom-0 w-64 bg-navy-900 text-white z-[80] flex flex-col shadow-xl"> <div className="mobile-menu-panel fixed top-0 right-0 bottom-0 w-64 bg-navy-900 text-white z-[80] flex flex-col shadow-xl">
<div className="flex items-center justify-between px-3 h-11 border-b border-navy-700"> <div className="flex items-center justify-between px-3 h-12 border-b border-navy-700">
<span className="font-semibold">{t('mobileMenu.menu')}</span> <span className="font-semibold">{t('mobileMenu.menu')}</span>
<button <button
onClick={onClose} onClick={onClose}

View file

@ -206,7 +206,6 @@ export function useHexagonSelection({
const params = new URLSearchParams({ const params = new URLSearchParams({
h3, h3,
resolution: res.toString(), resolution: res.toString(),
limit: '100',
offset: offset.toString(), offset: offset.toString(),
}); });
@ -250,7 +249,6 @@ export function useHexagonSelection({
try { try {
const params = new URLSearchParams({ const params = new URLSearchParams({
postcode, postcode,
limit: '100',
offset: offset.toString(), offset: offset.toString(),
}); });
if (focusAddress && offset === 0) { if (focusAddress && offset === 0) {

View file

@ -127,7 +127,7 @@ export function useLocationSearch(mode?: string) {
const controller = new AbortController(); const controller = new AbortController();
abortRef.current = controller; abortRef.current = controller;
try { try {
const params = new URLSearchParams({ q: trimmed, limit: '20' }); const params = new URLSearchParams({ q: trimmed });
if (mode) params.set('mode', mode); if (mode) params.set('mode', mode);
const res = await fetch( const res = await fetch(
`/api/places?${params}`, `/api/places?${params}`,

View file

@ -0,0 +1,59 @@
import { useEffect, useState } from 'react';
import type { NearbyStation, GeoPoint } from '../lib/nearby-stations';
import {
STATION_CATEGORIES,
selectNearbyStations,
stationSearchBounds,
} from '../lib/nearby-stations';
import type { POIResponse } from '../types';
import { apiUrl, assertOk, authHeaders, logNonAbortError } from '../lib/api';
function boundsParam(bounds: ReturnType<typeof stationSearchBounds>): string {
return `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`;
}
export function useNearbyStations(origin: GeoPoint | null) {
const [stations, setStations] = useState<NearbyStation[]>([]);
const [loading, setLoading] = useState(false);
const lat = origin?.lat;
const lon = origin?.lon;
useEffect(() => {
if (lat == null || lon == null) {
setStations([]);
setLoading(false);
return;
}
const controller = new AbortController();
setStations([]);
setLoading(true);
const originPoint = { lat, lon };
const bounds = stationSearchBounds(originPoint);
const params = new URLSearchParams({
bounds: boundsParam(bounds),
categories: STATION_CATEGORIES.join(','),
});
fetch(apiUrl('pois', params), authHeaders({ signal: controller.signal }))
.then((response) => {
assertOk(response, 'nearby stations');
return response.json() as Promise<POIResponse>;
})
.then((json) => {
if (!controller.signal.aborted) {
setStations(selectNearbyStations(json.pois ?? [], originPoint));
}
})
.catch((error) => logNonAbortError('Failed to fetch nearby stations', error))
.finally(() => {
if (!controller.signal.aborted) setLoading(false);
});
return () => controller.abort();
}, [lat, lon]);
return { stations, loading };
}

View file

@ -44,7 +44,7 @@ function getPoiIconUrlForPoi(poi: POI): string {
} }
function isBundledPoiIcon(url: string): boolean { function isBundledPoiIcon(url: string): boolean {
return url.startsWith('/assets/poi-icons/'); return url.startsWith('/assets/poi-icons/') || url.startsWith('data:image/svg+xml');
} }
function hasBundledPoiLogo(poi: POI): boolean { function hasBundledPoiLogo(poi: POI): boolean {

View file

@ -0,0 +1,80 @@
import { cleanup, fireEvent, render } from '@testing-library/react';
import { afterEach, describe, expect, it } from 'vitest';
import type { MutableRefObject } from 'react';
import { useRetainedScrollTop } from './useRetainedScrollTop';
function ScrollPane({
restoreKey,
savedScrollTopRef,
suspendSave,
}: {
restoreKey: string;
savedScrollTopRef: MutableRefObject<number>;
suspendSave: boolean;
}) {
const { scrollRef, onScroll } = useRetainedScrollTop<HTMLDivElement>({
restoreKey,
scrollTopRef: savedScrollTopRef,
suspendSave,
});
return (
<div ref={scrollRef} onScroll={onScroll} data-testid="pane">
<div style={{ height: 2000 }} />
</div>
);
}
describe('useRetainedScrollTop', () => {
afterEach(() => {
cleanup();
});
it('keeps the saved scroll offset while replacement content is loading', () => {
const savedScrollTopRef = { current: 0 };
const view = render(
<ScrollPane
restoreKey="area:a"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
const pane = view.getByTestId('pane');
pane.scrollTop = 360;
fireEvent.scroll(pane);
expect(savedScrollTopRef.current).toBe(360);
view.rerender(
<ScrollPane restoreKey="area:b" savedScrollTopRef={savedScrollTopRef} suspendSave />
);
pane.scrollTop = 0;
fireEvent.scroll(pane);
expect(savedScrollTopRef.current).toBe(360);
view.rerender(
<ScrollPane
restoreKey="area:b"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
expect(pane.scrollTop).toBe(360);
});
it('restores the saved offset when a pane remounts', () => {
const savedScrollTopRef = { current: 220 };
const view = render(
<ScrollPane
restoreKey="area:a"
savedScrollTopRef={savedScrollTopRef}
suspendSave={false}
/>
);
expect(view.getByTestId('pane').scrollTop).toBe(220);
});
});

View file

@ -0,0 +1,63 @@
import { useCallback, useLayoutEffect, useRef, type MutableRefObject, type UIEvent } from 'react';
interface UseRetainedScrollTopOptions {
restoreKey: string | null;
scrollTopRef?: MutableRefObject<number>;
suspendSave?: boolean;
}
export function useRetainedScrollTop<T extends HTMLElement>({
restoreKey,
scrollTopRef,
suspendSave = false,
}: UseRetainedScrollTopOptions) {
const fallbackScrollTopRef = useRef(0);
const savedScrollTopRef = scrollTopRef ?? fallbackScrollTopRef;
const nodeRef = useRef<T | null>(null);
const previousRestoreKeyRef = useRef(restoreKey);
const pendingRestoreTopRef = useRef<number | null>(null);
const suspendSaveRef = useRef(suspendSave);
suspendSaveRef.current = suspendSave;
const scrollRef = useCallback(
(node: T | null) => {
const previousNode = nodeRef.current;
if (!node && previousNode && !suspendSaveRef.current) {
savedScrollTopRef.current = previousNode.scrollTop;
}
nodeRef.current = node;
if (node) {
node.scrollTop = pendingRestoreTopRef.current ?? savedScrollTopRef.current;
}
},
[savedScrollTopRef]
);
const onScroll = useCallback(
(event: UIEvent<T>) => {
if (suspendSaveRef.current) return;
savedScrollTopRef.current = event.currentTarget.scrollTop;
},
[savedScrollTopRef]
);
useLayoutEffect(() => {
if (previousRestoreKeyRef.current !== restoreKey) {
previousRestoreKeyRef.current = restoreKey;
pendingRestoreTopRef.current = savedScrollTopRef.current;
}
const node = nodeRef.current;
const pendingRestoreTop = pendingRestoreTopRef.current;
if (!node || pendingRestoreTop == null) return;
node.scrollTop = pendingRestoreTop;
if (!suspendSave) {
pendingRestoreTopRef.current = null;
}
}, [restoreKey, savedScrollTopRef, suspendSave]);
return { scrollRef, onScroll };
}

View file

@ -842,6 +842,8 @@ const de: Translations = {
showAllStatsFallback: showAllStatsFallback:
'Wechseln Sie zu allen Immobilien, um dieses Gebiet ohne aktive Filter zu prüfen.', 'Wechseln Sie zu allen Immobilien, um dieses Gebiet ohne aktive Filter zu prüfen.',
showAllStats: 'Alle Immobilien anzeigen', showAllStats: 'Alle Immobilien anzeigen',
closestStations: 'Nächste Bahnhöfe',
noNearbyStations: 'Keine Bahn- oder U-Bahn-Station im Umkreis von 2 km',
closestBlockingFilters: 'Nächste Änderungen, um dieses Gebiet einzuschließen', closestBlockingFilters: 'Nächste Änderungen, um dieses Gebiet einzuschließen',
lowerMinTo: 'Minimum auf {{value}} senken', lowerMinTo: 'Minimum auf {{value}} senken',
raiseMaxTo: 'Maximum auf {{value}} erhöhen', raiseMaxTo: 'Maximum auf {{value}} erhöhen',

View file

@ -816,6 +816,8 @@ const en = {
showAllStatsFallback: showAllStatsFallback:
'Switch to all properties to inspect this area without the active filters.', 'Switch to all properties to inspect this area without the active filters.',
showAllStats: 'Show all properties', showAllStats: 'Show all properties',
closestStations: 'Closest stations',
noNearbyStations: 'No train or tube stations within 2km',
closestBlockingFilters: 'Closest changes to include this area', closestBlockingFilters: 'Closest changes to include this area',
lowerMinTo: 'Lower minimum to {{value}}', lowerMinTo: 'Lower minimum to {{value}}',
raiseMaxTo: 'Raise maximum to {{value}}', raiseMaxTo: 'Raise maximum to {{value}}',

View file

@ -848,6 +848,8 @@ const fr: Translations = {
showAllStatsFallback: showAllStatsFallback:
'Passez à toutes les propriétés pour inspecter cette zone sans les filtres actifs.', 'Passez à toutes les propriétés pour inspecter cette zone sans les filtres actifs.',
showAllStats: 'Afficher toutes les propriétés', showAllStats: 'Afficher toutes les propriétés',
closestStations: 'Stations les plus proches',
noNearbyStations: 'Aucune gare ou station de métro à moins de 2 km',
closestBlockingFilters: 'Modifications les plus proches pour inclure cette zone', closestBlockingFilters: 'Modifications les plus proches pour inclure cette zone',
lowerMinTo: 'Abaisser le minimum à {{value}}', lowerMinTo: 'Abaisser le minimum à {{value}}',
raiseMaxTo: 'Augmenter le maximum à {{value}}', raiseMaxTo: 'Augmenter le maximum à {{value}}',

View file

@ -806,6 +806,8 @@ const hi: Translations = {
showAllStatsFallback: showAllStatsFallback:
'सक्रिय फिल्टर के बिना इस क्षेत्र को देखने के लिए सभी संपत्तियों पर जाएं.', 'सक्रिय फिल्टर के बिना इस क्षेत्र को देखने के लिए सभी संपत्तियों पर जाएं.',
showAllStats: 'सभी संपत्तियां दिखाएं', showAllStats: 'सभी संपत्तियां दिखाएं',
closestStations: 'निकटतम स्टेशन',
noNearbyStations: '2 किमी के भीतर कोई ट्रेन या ट्यूब स्टेशन नहीं',
closestBlockingFilters: 'इस क्षेत्र को शामिल करने के निकटतम बदलाव', closestBlockingFilters: 'इस क्षेत्र को शामिल करने के निकटतम बदलाव',
lowerMinTo: 'न्यूनतम को {{value}} तक घटाएं', lowerMinTo: 'न्यूनतम को {{value}} तक घटाएं',
raiseMaxTo: 'अधिकतम को {{value}} तक बढ़ाएं', raiseMaxTo: 'अधिकतम को {{value}} तक बढ़ाएं',

View file

@ -830,6 +830,8 @@ const hu: Translations = {
showAllStatsFallback: showAllStatsFallback:
'Váltson az összes ingatlanra, hogy aktív szűrők nélkül tekintse át ezt a területet.', 'Váltson az összes ingatlanra, hogy aktív szűrők nélkül tekintse át ezt a területet.',
showAllStats: 'Összes ingatlan mutatása', showAllStats: 'Összes ingatlan mutatása',
closestStations: 'Legközelebbi állomások',
noNearbyStations: 'Nincs vonat- vagy metróállomás 2 km-en belül',
closestBlockingFilters: 'A terület bevonásához legközelebbi módosítások', closestBlockingFilters: 'A terület bevonásához legközelebbi módosítások',
lowerMinTo: 'Minimum csökkentése erre: {{value}}', lowerMinTo: 'Minimum csökkentése erre: {{value}}',
raiseMaxTo: 'Maximum növelése erre: {{value}}', raiseMaxTo: 'Maximum növelése erre: {{value}}',

View file

@ -775,6 +775,8 @@ const zh: Translations = {
showAllStatsHint: '筛选前这里有 {{count}} 处房产。切换到全部房产即可查看该区域。', showAllStatsHint: '筛选前这里有 {{count}} 处房产。切换到全部房产即可查看该区域。',
showAllStatsFallback: '切换到全部房产即可在不应用当前筛选条件的情况下查看该区域。', showAllStatsFallback: '切换到全部房产即可在不应用当前筛选条件的情况下查看该区域。',
showAllStats: '显示全部房产', showAllStats: '显示全部房产',
closestStations: '最近的车站',
noNearbyStations: '2 公里内没有火车站或地铁站',
closestBlockingFilters: '纳入该区域所需的最小调整', closestBlockingFilters: '纳入该区域所需的最小调整',
lowerMinTo: '将最小值降至 {{value}}', lowerMinTo: '将最小值降至 {{value}}',
raiseMaxTo: '将最大值提高至 {{value}}', raiseMaxTo: '将最大值提高至 {{value}}',

View file

@ -28,6 +28,17 @@ button:not(:disabled),
cursor: pointer; cursor: pointer;
} }
.area-pane-group-header {
box-shadow: inset 0 -1px 0 #e7e5e4;
transition:
background-color 0.2s ease,
color 0.2s ease;
}
html.dark .area-pane-group-header {
box-shadow: inset 0 -1px 0 #1e2d50;
}
/* Smooth theme transitions (scoped to avoid map performance issues) */ /* Smooth theme transitions (scoped to avoid map performance issues) */
body, body,
div, div,

View file

@ -132,6 +132,7 @@ export const POI_GROUP_COLORS: Record<string, [number, number, number]> = {
export const POI_CATEGORY_LOGOS: Record<string, string> = { export const POI_CATEGORY_LOGOS: Record<string, string> = {
Airport: '/assets/twemoji/2708.png', Airport: '/assets/twemoji/2708.png',
Aldi: '/assets/poi-icons/logos/aldi.svg', Aldi: '/assets/poi-icons/logos/aldi.svg',
'Allendale Co-operative Society': '/assets/poi-icons/logos/coop.svg',
Amazon: '/assets/poi-icons/brands_2024/amazon_fresh.svg', Amazon: '/assets/poi-icons/brands_2024/amazon_fresh.svg',
Asda: '/assets/poi-icons/logos/asda.svg', Asda: '/assets/poi-icons/logos/asda.svg',
'Asda Express': '/assets/poi-icons/logos/asda.svg', 'Asda Express': '/assets/poi-icons/logos/asda.svg',
@ -147,18 +148,26 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'Bus stop': '/assets/twemoji/1f68f.png', 'Bus stop': '/assets/twemoji/1f68f.png',
'Butcher & Fishmonger': '/assets/twemoji/1f969.png', 'Butcher & Fishmonger': '/assets/twemoji/1f969.png',
Centra: '/assets/poi-icons/logos/centra.svg', Centra: '/assets/poi-icons/logos/centra.svg',
'Central England Co-operative': '/assets/poi-icons/logos/coop.svg',
'Chelmsford Star Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'Clydebank Co-operative': '/assets/poi-icons/logos/coop.svg',
'Co-op': '/assets/poi-icons/logos/coop.svg', 'Co-op': '/assets/poi-icons/logos/coop.svg',
'Coniston Co-operative Society': '/assets/poi-icons/logos/coop.svg',
COOK: '/assets/poi-icons/brands_2024/cook.svg', COOK: '/assets/poi-icons/brands_2024/cook.svg',
'Convenience Store': '/assets/twemoji/1f3ea.png', 'Convenience Store': '/assets/twemoji/1f3ea.png',
Costco: '/assets/poi-icons/logos/costco.svg', Costco: '/assets/poi-icons/logos/costco.svg',
'Deli & Specialty': '/assets/twemoji/1f9c6.png', 'Deli & Specialty': '/assets/twemoji/1f9c6.png',
'Dunnes Stores': '/assets/poi-icons/brands_2024/dunnes_stores.svg', 'Dunnes Stores': '/assets/poi-icons/brands_2024/dunnes_stores.svg',
'East of England Co-operative': '/assets/poi-icons/logos/coop.svg',
Farmfoods: '/assets/poi-icons/brands_2023/supermarkets/farmfoods.svg', Farmfoods: '/assets/poi-icons/brands_2023/supermarkets/farmfoods.svg',
Ferry: '/assets/twemoji/26f4.png', Ferry: '/assets/twemoji/26f4.png',
Greengrocer: '/assets/twemoji/1f96c.png', Greengrocer: '/assets/twemoji/1f96c.png',
'Heart of England Co-operative': '/assets/poi-icons/logos/coop.svg',
'Heron Foods': '/assets/poi-icons/brands_2023/supermarkets/heron_foods.svg', 'Heron Foods': '/assets/poi-icons/brands_2023/supermarkets/heron_foods.svg',
Iceland: '/assets/poi-icons/brands_2024/iceland.svg', Iceland: '/assets/poi-icons/brands_2024/iceland.svg',
Lidl: '/assets/poi-icons/logos/lidl.svg', Lidl: '/assets/poi-icons/logos/lidl.svg',
'Langdale Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'Lincolnshire Co-operative': '/assets/poi-icons/logos/coop.svg',
Makro: '/assets/poi-icons/brands_2024/makro.svg', Makro: '/assets/poi-icons/brands_2024/makro.svg',
'M&S': '/assets/poi-icons/brands_2024/mns.svg', 'M&S': '/assets/poi-icons/brands_2024/mns.svg',
'M&S Clothing': '/assets/poi-icons/brands_2024/mns.svg', 'M&S Clothing': '/assets/poi-icons/brands_2024/mns.svg',
@ -166,6 +175,7 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'M&S Hospital': '/assets/poi-icons/brands_2024/mns.svg', 'M&S Hospital': '/assets/poi-icons/brands_2024/mns.svg',
'M&S MSA': '/assets/poi-icons/brands_2024/mns.svg', 'M&S MSA': '/assets/poi-icons/brands_2024/mns.svg',
'M&S Outlet': '/assets/poi-icons/brands_2024/mns.svg', 'M&S Outlet': '/assets/poi-icons/brands_2024/mns.svg',
'Midcounties Co-operative': '/assets/poi-icons/logos/coop.svg',
Morrisons: '/assets/poi-icons/logos/morrisons.svg', Morrisons: '/assets/poi-icons/logos/morrisons.svg',
'Morrisons Daily': '/assets/poi-icons/brands_2024/morrisons_daily.svg', 'Morrisons Daily': '/assets/poi-icons/brands_2024/morrisons_daily.svg',
'Off-Licence': '/assets/twemoji/1f377.png', 'Off-Licence': '/assets/twemoji/1f377.png',
@ -173,12 +183,16 @@ export const POI_CATEGORY_LOGOS: Record<string, string> = {
'Rail station': '/assets/twemoji/1f686.png', 'Rail station': '/assets/twemoji/1f686.png',
"Sainsbury's": '/assets/poi-icons/logos/sainsburys.svg', "Sainsbury's": '/assets/poi-icons/logos/sainsburys.svg',
"Sainsbury's Local": '/assets/poi-icons/brands_2024/sainsburys_local.svg', "Sainsbury's Local": '/assets/poi-icons/brands_2024/sainsburys_local.svg',
'Scottish Midland Co-operative': '/assets/poi-icons/logos/coop.svg',
Spar: '/assets/poi-icons/logos/spar.svg', Spar: '/assets/poi-icons/logos/spar.svg',
Supermarket: '/assets/twemoji/1f6d2.png', Supermarket: '/assets/twemoji/1f6d2.png',
'Tamworth Co-operative Society': '/assets/poi-icons/logos/coop.svg',
Tesco: '/assets/poi-icons/logos/tesco.svg', Tesco: '/assets/poi-icons/logos/tesco.svg',
'Tesco Express': '/assets/poi-icons/logos/tesco_express.svg', 'Tesco Express': '/assets/poi-icons/logos/tesco_express.svg',
'Tesco Extra': '/assets/poi-icons/logos/tesco_extra.svg', 'Tesco Extra': '/assets/poi-icons/logos/tesco_extra.svg',
'Taxi rank': '/assets/twemoji/1f695.png', 'Taxi rank': '/assets/twemoji/1f695.png',
'The Radstock Co-operative Society': '/assets/poi-icons/logos/coop.svg',
'The Southern Co-operative': '/assets/poi-icons/logos/coop.svg',
'The Food Warehouse': '/assets/poi-icons/logos/the_food_warehouse.png', 'The Food Warehouse': '/assets/poi-icons/logos/the_food_warehouse.png',
'Tube station': '/assets/poi-icons/public_transport/london_tube.svg', 'Tube station': '/assets/poi-icons/public_transport/london_tube.svg',
Waitrose: '/assets/poi-icons/logos/waitrose.svg', Waitrose: '/assets/poi-icons/logos/waitrose.svg',

View file

@ -71,7 +71,7 @@ describe('map utilities', () => {
expect(enumIndexToColor(ENUM_PALETTE.length, ENUM_PALETTE)).toEqual(ENUM_PALETTE[0]); expect(enumIndexToColor(ENUM_PALETTE.length, ENUM_PALETTE)).toEqual(ENUM_PALETTE[0]);
}); });
it('resolves POI category logos and rejects unknown icon categories', () => { it('resolves POI category logos and generates a fallback for unknown chains', () => {
expect(getPoiIconUrl('Waitrose', '🛒')).toBe('/assets/poi-icons/logos/waitrose.svg'); expect(getPoiIconUrl('Waitrose', '🛒')).toBe('/assets/poi-icons/logos/waitrose.svg');
expect(getPoiIconUrl('Iceland', '🛒', 'The Food Warehouse')).toBe( expect(getPoiIconUrl('Iceland', '🛒', 'The Food Warehouse')).toBe(
'/assets/poi-icons/logos/the_food_warehouse.png' '/assets/poi-icons/logos/the_food_warehouse.png'
@ -83,8 +83,8 @@ describe('map utilities', () => {
expect(getPoiIconUrl('M&S', '🛒', undefined, 'M&S Simply Food')).toBe( expect(getPoiIconUrl('M&S', '🛒', undefined, 'M&S Simply Food')).toBe(
'/assets/poi-icons/visuals/mns.svg' '/assets/poi-icons/visuals/mns.svg'
); );
expect(() => getPoiIconUrl('Unknown category', '🛒')).toThrow( expect(getPoiIconUrl('Tian Tian', '🛒')).toMatch(
"Missing POI icon for category 'Unknown category'" /^data:image\/svg\+xml;charset=utf-8,/
); );
}); });

View file

@ -309,9 +309,67 @@ function inferPoiIconCategory(category: string, name?: string): string | undefin
} }
} }
export function getPoiIconUrl( const GENERATED_POI_LOGO_COLORS: [string, string][] = [
['#0f766e', '#ccfbf1'],
['#1d4ed8', '#dbeafe'],
['#b45309', '#fef3c7'],
['#be123c', '#ffe4e6'],
['#6d28d9', '#ede9fe'],
['#047857', '#d1fae5'],
['#9d174d', '#fce7f3'],
['#334155', '#e2e8f0'],
];
const generatedPoiLogoCache = new Map<string, string>();
function hashLabel(label: string): number {
let hash = 0;
for (let i = 0; i < label.length; i += 1) {
hash = (hash * 31 + label.charCodeAt(i)) >>> 0;
}
return hash;
}
function escapeSvgText(text: string): string {
return text
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;');
}
function getPoiLogoInitials(label: string): string {
const words = label.match(/[A-Za-z0-9]+/g) ?? [];
const significantWords = words.filter(
(word) => !['and', 'of', 'the'].includes(word.toLowerCase())
);
const selectedWords = significantWords.length > 0 ? significantWords : words;
if (selectedWords.length === 0) return 'POI';
if (selectedWords.length === 1) return selectedWords[0].slice(0, 3).toUpperCase();
return selectedWords
.slice(0, 3)
.map((word) => word[0])
.join('')
.toUpperCase();
}
function getGeneratedPoiLogoUrl(label: string): string {
const key = label.trim() || 'POI';
const cached = generatedPoiLogoCache.get(key);
if (cached) return cached;
const [background, foreground] = GENERATED_POI_LOGO_COLORS[
hashLabel(key) % GENERATED_POI_LOGO_COLORS.length
];
const initials = escapeSvgText(getPoiLogoInitials(key));
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="256" height="256" viewBox="0 0 256 256"><rect width="256" height="256" rx="48" fill="${background}"/><text x="128" y="144" text-anchor="middle" font-family="Inter,Arial,sans-serif" font-size="82" font-weight="800" fill="${foreground}">${initials}</text></svg>`;
const url = `data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}`;
generatedPoiLogoCache.set(key, url);
return url;
}
export function getPoiCategoryLogoUrl(
category: string, category: string,
_emoji: string,
iconCategory?: string, iconCategory?: string,
name?: string name?: string
): string { ): string {
@ -319,11 +377,16 @@ export function getPoiIconUrl(
if (resolvedIconCategory && POI_CATEGORY_LOGOS[resolvedIconCategory]) { if (resolvedIconCategory && POI_CATEGORY_LOGOS[resolvedIconCategory]) {
return POI_CATEGORY_LOGOS[resolvedIconCategory]; return POI_CATEGORY_LOGOS[resolvedIconCategory];
} }
const categoryLogo = POI_CATEGORY_LOGOS[category]; return POI_CATEGORY_LOGOS[category] ?? getGeneratedPoiLogoUrl(resolvedIconCategory || category);
if (!categoryLogo) { }
throw new Error(`Missing POI icon for category '${category}'`);
} export function getPoiIconUrl(
return categoryLogo; category: string,
_emoji: string,
iconCategory?: string,
name?: string
): string {
return getPoiCategoryLogoUrl(category, iconCategory, name);
} }
/** Look up a discrete color from the enum palette by index (wraps if > palette size). */ /** Look up a discrete color from the enum palette by index (wraps if > palette size). */

View file

@ -0,0 +1,65 @@
import { describe, expect, it } from 'vitest';
import type { POI } from '../types';
import {
formatStationDistance,
selectNearbyStations,
stationSearchBounds,
} from './nearby-stations';
function poi(name: string, lat: number, lng: number): POI {
return {
id: name,
name,
category: 'Rail station',
icon_category: 'Rail station',
group: 'Public Transport',
lat,
lng,
emoji: '',
};
}
describe('selectNearbyStations', () => {
const origin = { lat: 51.5, lon: -0.1 };
it('returns every station within 1km when any station is inside 1km', () => {
const stations = selectNearbyStations(
[poi('Close A', 51.501, -0.1), poi('Close B', 51.502, -0.1), poi('Fallback', 51.511, -0.1)],
origin
);
expect(stations.map((station) => station.name)).toEqual(['Close A', 'Close B']);
});
it('falls back to the nearest three stations within 2km when none are inside 1km', () => {
const stations = selectNearbyStations(
[
poi('Fourth', 51.516, -0.1),
poi('First', 51.51, -0.1),
poi('Outside', 51.53, -0.1),
poi('Third', 51.514, -0.1),
poi('Second', 51.512, -0.1),
],
origin
);
expect(stations.map((station) => station.name)).toEqual(['First', 'Second', 'Third']);
});
it('formats sub-kilometre and kilometre distances', () => {
expect(formatStationDistance(0.234)).toBe('234m');
expect(formatStationDistance(1.234)).toBe('1.2km');
});
});
describe('stationSearchBounds', () => {
it('builds a box around the origin', () => {
const bounds = stationSearchBounds({ lat: 51.5, lon: -0.1 });
expect(bounds.south).toBeLessThan(51.5);
expect(bounds.north).toBeGreaterThan(51.5);
expect(bounds.west).toBeLessThan(-0.1);
expect(bounds.east).toBeGreaterThan(-0.1);
});
});

View file

@ -0,0 +1,71 @@
import type { Bounds, POI } from '../types';
export const STATION_CATEGORIES = ['Rail station', 'Tube station'] as const;
export const STATION_SEARCH_RADIUS_KM = 2;
const PRIMARY_STATION_RADIUS_KM = 1;
const FALLBACK_STATION_LIMIT = 3;
const EARTH_RADIUS_KM = 6371;
const KM_PER_DEGREE_LAT = 111.32;
export interface NearbyStation extends POI {
distanceKm: number;
}
export interface GeoPoint {
lat: number;
lon: number;
}
function toRadians(value: number): number {
return (value * Math.PI) / 180;
}
export function distanceKm(a: GeoPoint, b: GeoPoint): number {
const dLat = toRadians(b.lat - a.lat);
const dLon = toRadians(b.lon - a.lon);
const lat1 = toRadians(a.lat);
const lat2 = toRadians(b.lat);
const h = Math.sin(dLat / 2) ** 2 + Math.cos(lat1) * Math.cos(lat2) * Math.sin(dLon / 2) ** 2;
return 2 * EARTH_RADIUS_KM * Math.asin(Math.sqrt(h));
}
export function stationSearchBounds(origin: GeoPoint, radiusKm = STATION_SEARCH_RADIUS_KM): Bounds {
const latDelta = radiusKm / KM_PER_DEGREE_LAT;
const cosLat = Math.max(Math.cos(toRadians(origin.lat)), 0.01);
const lonDelta = radiusKm / (KM_PER_DEGREE_LAT * cosLat);
return {
south: origin.lat - latDelta,
west: origin.lon - lonDelta,
north: origin.lat + latDelta,
east: origin.lon + lonDelta,
};
}
export function selectNearbyStations(pois: POI[], origin: GeoPoint): NearbyStation[] {
const stations = pois
.map((poi) => ({
...poi,
distanceKm: distanceKm(origin, { lat: poi.lat, lon: poi.lng }),
}))
.filter((poi) => poi.distanceKm <= STATION_SEARCH_RADIUS_KM)
.sort((a, b) => a.distanceKm - b.distanceKm || a.name.localeCompare(b.name));
const withinPrimaryRadius = stations.filter(
(station) => station.distanceKm <= PRIMARY_STATION_RADIUS_KM
);
return withinPrimaryRadius.length > 0
? withinPrimaryRadius
: stations.slice(0, FALLBACK_STATION_LIMIT);
}
export function formatStationDistance(distanceKmValue: number): string {
if (distanceKmValue < 1) {
return `${Math.round(distanceKmValue * 1000)}m`;
}
return `${distanceKmValue.toFixed(1)}km`;
}

View file

@ -202,7 +202,6 @@ export interface Property {
export interface PropertyListResponse { export interface PropertyListResponse {
properties: Property[]; properties: Property[];
total: number; total: number;
limit: number;
offset: number; offset: number;
truncated: boolean; truncated: boolean;
} }

View file

@ -15,7 +15,7 @@ def test_transform_grocery_retail_points_outputs_chain_categories():
} }
) )
pois = transform_grocery_retail_points(raw) pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select( assert pois.select(
"id", "name", "category", "icon_category", "group", "emoji" "id", "name", "category", "icon_category", "group", "emoji"
@ -69,7 +69,7 @@ def test_transform_grocery_retail_points_keeps_fascia_icon_category():
} }
) )
pois = transform_grocery_retail_points(raw) pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select("category", "icon_category").to_dicts() == [ assert pois.select("category", "icon_category").to_dicts() == [
{"category": "Tesco", "icon_category": "Tesco Express"}, {"category": "Tesco", "icon_category": "Tesco Express"},
@ -96,7 +96,7 @@ def test_transform_grocery_retail_points_accepts_base_fascias():
} }
) )
pois = transform_grocery_retail_points(raw) pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois.select("category", "icon_category").to_dicts() == [ assert pois.select("category", "icon_category").to_dicts() == [
{"category": "Aldi", "icon_category": "Aldi"}, {"category": "Aldi", "icon_category": "Aldi"},
@ -118,6 +118,29 @@ def test_transform_grocery_retail_points_drops_invalid_rows():
} }
) )
pois = transform_grocery_retail_points(raw) pois = transform_grocery_retail_points(raw, min_chain_locations=1)
assert pois["category"].to_list() == ["Waitrose"] assert pois["category"].to_list() == ["Waitrose"]
def test_transform_grocery_retail_points_includes_unmapped_chains_with_five_locations():
raw = pl.DataFrame(
{
"id": list(range(1, 10)),
"retailer": ["Tian Tian"] * 5 + ["Corner Shop"] * 4,
"fascia": ["Tian Tian Market"] * 5 + ["Corner Shop"] * 4,
"store_name": [f"Store {i}" for i in range(1, 10)],
"long_wgs": [-0.1] * 9,
"lat_wgs": [51.5] * 9,
}
)
pois = transform_grocery_retail_points(raw)
assert pois.select("id", "category", "icon_category").to_dicts() == [
{"id": "glx-1", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-2", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-3", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-4", "category": "Tian Tian", "icon_category": "Tian Tian"},
{"id": "glx-5", "category": "Tian Tian", "icon_category": "Tian Tian"},
]

View file

@ -5,7 +5,6 @@ import polars as pl
from pipeline.utils.england_geometry import in_england_mask from pipeline.utils.england_geometry import in_england_mask
DROP_CATEGORIES = { DROP_CATEGORIES = {
# Street furniture & infrastructure # Street furniture & infrastructure
"amenity/advice", "amenity/advice",
@ -1165,49 +1164,44 @@ COOP_RETAILERS = {
"The Southern Co-operative", "The Southern Co-operative",
} }
GROCERY_RETAILER_DISPLAY_NAMES: dict[str, str] = { MIN_GROCERY_CHAIN_LOCATIONS = 5
"Aldi": "Aldi",
"Asda": "Asda", GROCERY_RETAILER_DISPLAY_NAME_OVERRIDES: dict[str, str] = {
"Booths": "Booths",
"Budgens": "Budgens",
"Centra": "Centra",
"Cook": "COOK", "Cook": "COOK",
"Costco": "Costco",
"Dunnes Stores": "Dunnes Stores",
"Farmfoods": "Farmfoods",
"Heron": "Heron Foods", "Heron": "Heron Foods",
"Iceland": "Iceland",
"Lidl": "Lidl",
"Makro": "Makro",
"Marks and Spencer": "M&S", "Marks and Spencer": "M&S",
"Morrisons": "Morrisons",
"Planet Organic": "Planet Organic",
"Sainsburys": "Sainsbury's", "Sainsburys": "Sainsbury's",
"Spar": "Spar", "The Co-operative Group": "Co-op",
"Tesco": "Tesco",
"Waitrose": "Waitrose",
"Whole Foods Market": "Whole Foods Market",
**{retailer: "Co-op" for retailer in COOP_RETAILERS},
} }
GROCERY_FASCIA_ICON_NAMES: dict[str, str] = { GROCERY_FASCIA_ICON_NAMES: dict[str, str] = {
**GROCERY_RETAILER_DISPLAY_NAMES, "Aldi": "Aldi",
"Aldi Local": "Aldi", "Aldi Local": "Aldi",
"Asda": "Asda",
"Asda Express": "Asda Express", "Asda Express": "Asda Express",
"Asda Living": "Asda Living", "Asda Living": "Asda Living",
"Asda PFS": "Asda PFS", "Asda PFS": "Asda",
"Asda Supercentre": "Asda Supercentre", "Asda Supercentre": "Asda Supercentre",
"Asda Supermarket": "Asda Supermarket", "Asda Supermarket": "Asda Supermarket",
"Asda Superstore": "Asda Superstore", "Asda Superstore": "Asda Superstore",
"Booths": "Booths",
"Budgens": "Budgens",
"Centra": "Centra",
"Cooltrader": "Heron Foods", "Cooltrader": "Heron Foods",
"Co-op Food": "Co-op", "Co-op Food": "Co-op",
"Cook": "COOK", "Cook": "COOK",
"Costco": "Costco",
"Dunnes Stores": "Dunnes Stores",
"Eurospar": "Spar", "Eurospar": "Spar",
"Eurospar PFS": "Spar", "Eurospar PFS": "Spar",
"Farmfoods": "Farmfoods",
"Heron": "Heron Foods", "Heron": "Heron Foods",
"Iceland": "Iceland",
"Lidl": "Lidl",
"Little Waitrose": "Little Waitrose", "Little Waitrose": "Little Waitrose",
"Little Waitrose Shell": "Little Waitrose", "Little Waitrose Shell": "Little Waitrose",
"Makro": "Makro",
"Marks and Spencer": "M&S", "Marks and Spencer": "M&S",
"Marks and Spencer BP": "M&S Food", "Marks and Spencer BP": "M&S Food",
"Marks and Spencer Clothing": "M&S Clothing", "Marks and Spencer Clothing": "M&S Clothing",
@ -1221,34 +1215,36 @@ GROCERY_FASCIA_ICON_NAMES: dict[str, str] = {
"Marks and Spencer Travel SF": "M&S Food", "Marks and Spencer Travel SF": "M&S Food",
"Morrisons Daily": "Morrisons Daily", "Morrisons Daily": "Morrisons Daily",
"Morrisons Select": "Morrisons", "Morrisons Select": "Morrisons",
"Planet Organic": "Planet Organic",
"Sainsbury's Local": "Sainsbury's Local", "Sainsbury's Local": "Sainsbury's Local",
"Sainsburys": "Sainsbury's", "Sainsburys": "Sainsbury's",
"Sainsburys Local": "Sainsbury's Local", "Sainsburys Local": "Sainsbury's Local",
"Spar": "Spar",
"Spar PFS": "Spar", "Spar PFS": "Spar",
"Tesco": "Tesco",
"Tesco Express": "Tesco Express", "Tesco Express": "Tesco Express",
"Tesco Express Esso": "Tesco Express", "Tesco Express Esso": "Tesco Express",
"Tesco Extra": "Tesco Extra", "Tesco Extra": "Tesco Extra",
"The Co-operative Food": "Co-op", "The Co-operative Food": "Co-op",
"The Co-operative Food PFS": "Co-op", "The Co-operative Food PFS": "Co-op",
"The Food Warehouse": "The Food Warehouse", "The Food Warehouse": "The Food Warehouse",
"Waitrose": "Waitrose",
"Waitrose MSA": "Waitrose", "Waitrose MSA": "Waitrose",
"Whole Foods Market": "Whole Foods Market",
} }
def normalize_grocery_retailer(retailer: str | None) -> str: def normalize_grocery_retailer(retailer: str | None) -> str:
if retailer is None: if retailer is None:
return "" return ""
display_name = GROCERY_RETAILER_DISPLAY_NAMES.get(retailer) retailer = retailer.strip()
if display_name is None: return GROCERY_RETAILER_DISPLAY_NAME_OVERRIDES.get(retailer, retailer)
raise ValueError(f"Missing grocery retailer display name for {retailer!r}")
return display_name
def normalize_grocery_icon_category(fascia: str | None, retailer: str | None) -> str: def normalize_grocery_icon_category(fascia: str | None, retailer: str | None) -> str:
if fascia: if fascia:
icon_name = GROCERY_FASCIA_ICON_NAMES.get(fascia) icon_name = GROCERY_FASCIA_ICON_NAMES.get(fascia.strip())
if icon_name is None: if icon_name is not None:
raise ValueError(f"Missing grocery fascia icon name for {fascia!r}")
return icon_name return icon_name
return normalize_grocery_retailer(retailer) return normalize_grocery_retailer(retailer)
@ -1256,6 +1252,7 @@ def normalize_grocery_icon_category(fascia: str | None, retailer: str | None) ->
def transform_grocery_retail_points( def transform_grocery_retail_points(
grocery_df: pl.DataFrame, grocery_df: pl.DataFrame,
boundary_path: Path | None = None, boundary_path: Path | None = None,
min_chain_locations: int = MIN_GROCERY_CHAIN_LOCATIONS,
) -> pl.DataFrame: ) -> pl.DataFrame:
"""Convert GEOLYTIX Grocery Retail Points into the POI parquet schema.""" """Convert GEOLYTIX Grocery Retail Points into the POI parquet schema."""
required = {"id", "retailer", "fascia", "store_name", "long_wgs", "lat_wgs"} required = {"id", "retailer", "fascia", "store_name", "long_wgs", "lat_wgs"}
@ -1272,6 +1269,11 @@ def transform_grocery_retail_points(
pl.col("lat_wgs").cast(pl.Float64).alias("lat"), pl.col("lat_wgs").cast(pl.Float64).alias("lat"),
pl.col("long_wgs").cast(pl.Float64).alias("lng"), pl.col("long_wgs").cast(pl.Float64).alias("lng"),
) )
.with_columns(
pl.col("retailer").str.strip_chars(),
pl.col("fascia").str.strip_chars(),
pl.col("store_name").str.strip_chars(),
)
.drop_nulls(["id", "retailer", "lat", "lng"]) .drop_nulls(["id", "retailer", "lat", "lng"])
.filter(pl.col("retailer").str.len_chars() > 0) .filter(pl.col("retailer").str.len_chars() > 0)
) )
@ -1284,6 +1286,14 @@ def transform_grocery_retail_points(
) )
df = df.filter(pl.Series(mask)) df = df.filter(pl.Series(mask))
eligible_retailers = (
df.group_by("retailer")
.len()
.filter(pl.col("len") >= min_chain_locations)
.select("retailer")
)
df = df.join(eligible_retailers, on="retailer", how="semi")
return df.with_columns( return df.with_columns(
pl.concat_str([pl.lit("glx-"), pl.col("id")]).alias("id"), pl.concat_str([pl.lit("glx-"), pl.col("id")]).alias("id"),
pl.coalesce(["store_name", "fascia", "retailer"]) pl.coalesce(["store_name", "fascia", "retailer"])

2
property-data2/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore

View file

@ -13,12 +13,10 @@ pub const GRID_CELL_SIZE: f32 = 0.01;
pub const MAX_CELLS_PER_REQUEST: usize = 200000; pub const MAX_CELLS_PER_REQUEST: usize = 200000;
pub const MAX_POIS_PER_REQUEST: usize = 3000; pub const MAX_POIS_PER_REQUEST: usize = 3000;
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100; pub const PROPERTIES_LIMIT: usize = 100;
pub const DEFAULT_ACTUAL_LISTINGS_LIMIT: usize = 500; pub const ACTUAL_LISTINGS_LIMIT: usize = 500;
pub const MAX_ACTUAL_LISTINGS_LIMIT: usize = 2000; pub const PLACES_LIMIT: usize = 20;
pub const MAX_PLACES_LIMIT: usize = 20; pub const PRICE_HISTORY_POINTS_LIMIT: usize = 5000;
pub const DEFAULT_PLACES_LIMIT: usize = 7;
pub const MAX_PRICE_HISTORY_POINTS: usize = 5000;
pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02; pub const POSTCODE_SEARCH_OFFSET: f64 = 0.02;
pub const AI_FILTERS_MAX_TOKENS: usize = 2000; pub const AI_FILTERS_MAX_TOKENS: usize = 2000;

View file

@ -30,6 +30,16 @@ const GROCERY_DASHBOARD_CATEGORIES: &[&str] = &[
"Budgens", "Budgens",
"Centra", "Centra",
"Co-op", "Co-op",
"Central England Co-operative",
"Chelmsford Star Co-operative Society",
"East of England Co-operative",
"Heart of England Co-operative",
"Lincolnshire Co-operative",
"Midcounties Co-operative",
"Scottish Midland Co-operative",
"Tamworth Co-operative Society",
"The Radstock Co-operative Society",
"The Southern Co-operative",
"COOK", "COOK",
"Costco", "Costco",
"Dunnes Stores", "Dunnes Stores",

View file

@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use tracing::info; use tracing::info;
use crate::api_error::ApiError; use crate::api_error::ApiError;
use crate::consts::{DEFAULT_ACTUAL_LISTINGS_LIMIT, MAX_ACTUAL_LISTINGS_LIMIT}; use crate::consts::ACTUAL_LISTINGS_LIMIT;
use crate::data::ActualListing; use crate::data::ActualListing;
use crate::features::property_level_feature_names; use crate::features::property_level_feature_names;
use crate::parsing::{ use crate::parsing::{
@ -24,9 +24,6 @@ pub struct ActualListingsParams {
filters: Option<String>, filters: Option<String>,
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max` /// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`
travel: Option<String>, travel: Option<String>,
/// Page size — defaults to DEFAULT_ACTUAL_LISTINGS_LIMIT, capped at
/// MAX_ACTUAL_LISTINGS_LIMIT.
limit: Option<usize>,
/// Number of results to skip. Defaults to 0. /// Number of results to skip. Defaults to 0.
offset: Option<usize>, offset: Option<usize>,
} }
@ -35,7 +32,6 @@ pub struct ActualListingsParams {
pub struct ActualListingsResponse { pub struct ActualListingsResponse {
pub listings: Vec<ActualListing>, pub listings: Vec<ActualListing>,
pub total: usize, pub total: usize,
pub limit: usize,
pub offset: usize, pub offset: usize,
pub truncated: bool, pub truncated: bool,
} }
@ -45,16 +41,12 @@ pub async fn get_actual_listings(
Query(params): Query<ActualListingsParams>, Query(params): Query<ActualListingsParams>,
) -> Result<Json<ActualListingsResponse>, ApiError> { ) -> Result<Json<ActualListingsResponse>, ApiError> {
let state = shared.load_state(); let state = shared.load_state();
let limit = params let limit = ACTUAL_LISTINGS_LIMIT;
.limit
.unwrap_or(DEFAULT_ACTUAL_LISTINGS_LIMIT)
.min(MAX_ACTUAL_LISTINGS_LIMIT);
let offset = params.offset.unwrap_or(0); let offset = params.offset.unwrap_or(0);
let Some(actual_listings) = state.actual_listings.clone() else { let Some(actual_listings) = state.actual_listings.clone() else {
return Ok(Json(ActualListingsResponse { return Ok(Json(ActualListingsResponse {
listings: Vec::new(), listings: Vec::new(),
total: 0, total: 0,
limit,
offset, offset,
truncated: false, truncated: false,
})); }));
@ -162,7 +154,6 @@ pub async fn get_actual_listings(
Ok(ActualListingsResponse { Ok(ActualListingsResponse {
listings, listings,
total: total_matching, total: total_matching,
limit,
offset, offset,
truncated, truncated,
}) })

View file

@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
use tracing::info; use tracing::info;
use crate::api_error::ApiError; use crate::api_error::ApiError;
use crate::consts::{DEFAULT_PLACES_LIMIT, MAX_PLACES_LIMIT}; use crate::consts::PLACES_LIMIT;
use crate::data::{normalize_search_text, slugify}; use crate::data::{normalize_search_text, slugify};
use crate::state::SharedState; use crate::state::SharedState;
@ -42,7 +42,6 @@ pub struct PlacesResponse {
#[allow(clippy::min_ident_chars)] #[allow(clippy::min_ident_chars)]
pub struct PlacesParams { pub struct PlacesParams {
q: String, q: String,
limit: Option<usize>,
/// If set, only return places that have travel time data for this mode. /// If set, only return places that have travel time data for this mode.
mode: Option<String>, mode: Option<String>,
} }
@ -105,10 +104,7 @@ pub async fn get_places(
params.q params.q
}; };
let limit = params let limit = PLACES_LIMIT;
.limit
.unwrap_or(DEFAULT_PLACES_LIMIT)
.min(MAX_PLACES_LIMIT);
let mode_filter = params.mode; let mode_filter = params.mode;
let places = tokio::task::spawn_blocking(move || { let places = tokio::task::spawn_blocking(move || {

View file

@ -8,7 +8,7 @@ use serde::Deserialize;
use tracing::{info, warn}; use tracing::{info, warn};
use crate::auth::OptionalUser; use crate::auth::OptionalUser;
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, POSTCODE_SEARCH_OFFSET}; use crate::consts::{POSTCODE_SEARCH_OFFSET, PROPERTIES_LIMIT};
use crate::licensing::{check_license_point, resolve_share_code}; use crate::licensing::{check_license_point, resolve_share_code};
use crate::parsing::{parse_filters_with_poi, row_passes_filters, row_passes_poi_filters}; use crate::parsing::{parse_filters_with_poi, row_passes_filters, row_passes_poi_filters};
use crate::state::SharedState; use crate::state::SharedState;
@ -24,7 +24,6 @@ pub struct PostcodePropertiesParams {
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`. /// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`.
/// Optional min:max applies as a filter (exclude properties outside range). /// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>, pub travel: Option<String>,
pub limit: Option<usize>,
pub offset: Option<usize>, pub offset: Option<usize>,
/// Exact address to rank first when opening properties from address search. /// Exact address to rank first when opening properties from address search.
pub focus_address: Option<String>, pub focus_address: Option<String>,
@ -151,7 +150,7 @@ pub async fn get_postcode_properties(
}); });
let total = matching_rows.len(); let total = matching_rows.len();
let limit = params.limit.unwrap_or(DEFAULT_PROPERTIES_LIMIT); let limit = PROPERTIES_LIMIT;
let page_offset = params.offset.unwrap_or(0); let page_offset = params.offset.unwrap_or(0);
let truncated = total > page_offset + limit; let truncated = total > page_offset + limit;
@ -186,7 +185,6 @@ pub async fn get_postcode_properties(
Ok(PropertyListResponse { Ok(PropertyListResponse {
properties, properties,
total, total,
limit,
offset: page_offset, offset: page_offset,
truncated, truncated,
}) })

View file

@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
use tracing::{info, warn}; use tracing::{info, warn};
use crate::auth::OptionalUser; use crate::auth::OptionalUser;
use crate::consts::DEFAULT_PROPERTIES_LIMIT; use crate::consts::PROPERTIES_LIMIT;
use crate::data::RenovationEvent; use crate::data::RenovationEvent;
use crate::licensing::{check_license_bounds, resolve_share_code}; use crate::licensing::{check_license_bounds, resolve_share_code};
use crate::parsing::{ use crate::parsing::{
@ -29,7 +29,6 @@ pub struct HexagonPropertiesParams {
/// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`. /// Pipe-separated travel time entries: `mode:slug|mode:slug:min:max`.
/// Optional min:max applies as a filter (exclude properties outside range). /// Optional min:max applies as a filter (exclude properties outside range).
pub travel: Option<String>, pub travel: Option<String>,
pub limit: Option<usize>,
pub offset: Option<usize>, pub offset: Option<usize>,
/// Share-link code; grants bbox-scoped access for unlicensed users. /// Share-link code; grants bbox-scoped access for unlicensed users.
pub share: Option<String>, pub share: Option<String>,
@ -69,7 +68,6 @@ pub struct Property {
pub struct PropertyListResponse { pub struct PropertyListResponse {
pub properties: Vec<Property>, pub properties: Vec<Property>,
pub total: usize, pub total: usize,
pub limit: usize,
pub offset: usize, pub offset: usize,
pub truncated: bool, pub truncated: bool,
} }
@ -276,7 +274,7 @@ pub async fn get_hexagon_properties(
matching_rows.sort_unstable_by_key(|&row| state.data.address(row).trim().is_empty()); matching_rows.sort_unstable_by_key(|&row| state.data.address(row).trim().is_empty());
let total = matching_rows.len(); let total = matching_rows.len();
let limit = params.limit.unwrap_or(DEFAULT_PROPERTIES_LIMIT); let limit = PROPERTIES_LIMIT;
let offset = params.offset.unwrap_or(0); let offset = params.offset.unwrap_or(0);
let truncated = total > offset + limit; let truncated = total > offset + limit;
@ -312,7 +310,6 @@ pub async fn get_hexagon_properties(
Ok(PropertyListResponse { Ok(PropertyListResponse {
properties, properties,
total, total,
limit,
offset, offset,
truncated, truncated,
}) })

View file

@ -4,7 +4,7 @@ use metrics::counter;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use tracing::error; use tracing::error;
use crate::consts::MAX_PRICE_HISTORY_POINTS; use crate::consts::PRICE_HISTORY_POINTS_LIMIT;
use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData}; use crate::data::{FeatureStats, PostcodePoiMetrics, PropertyData};
use super::hexagon_stats::{EnumFeatureStats, HistogramStats, NumericFeatureStats, PricePoint}; use super::hexagon_stats::{EnumFeatureStats, HistogramStats, NumericFeatureStats, PricePoint};
@ -32,9 +32,9 @@ pub fn extract_price_history(
} }
}) })
.collect(); .collect();
if points.len() > MAX_PRICE_HISTORY_POINTS { if points.len() > PRICE_HISTORY_POINTS_LIMIT {
let step = points.len() as f64 / MAX_PRICE_HISTORY_POINTS as f64; let step = points.len() as f64 / PRICE_HISTORY_POINTS_LIMIT as f64;
points = (0..MAX_PRICE_HISTORY_POINTS) points = (0..PRICE_HISTORY_POINTS_LIMIT)
.map(|i| { .map(|i| {
let idx = (i as f64 * step) as usize; let idx = (i as f64 * step) as usize;
PricePoint { PricePoint {