diff --git a/.forgejo/workflows/docker-publish.yml b/.forgejo/workflows/docker-publish.yml index 365476a..df3e5dc 100644 --- a/.forgejo/workflows/docker-publish.yml +++ b/.forgejo/workflows/docker-publish.yml @@ -26,6 +26,13 @@ jobs: - name: Download arcgis data for finder run: uv run python -m pipeline.download.arcgis --output property-data/arcgis_data.parquet + - name: Install Docker CLI + run: | + ARCH=$(uname -m) + curl -fsSL "https://download.docker.com/linux/static/stable/${ARCH}/docker-27.5.1.tgz" \ + | tar xz --strip-components=1 -C /usr/local/bin docker/docker + docker --version + - name: Set up Docker Buildx uses: https://github.com/docker/setup-buildx-action@v3 diff --git a/finder/constants.py b/finder/constants.py index 2985486..f10d5da 100644 --- a/finder/constants.py +++ b/finder/constants.py @@ -4,12 +4,13 @@ from pathlib import Path ARCGIS_PATH = os.environ.get("ARCGIS_PATH", "/data/arcgis_data.parquet") DATA_DIR = Path("/app/data") PAGE_SIZE = 24 -DELAY_BETWEEN_PAGES = 0.5 -DELAY_BETWEEN_OUTCODES = 1.0 +DELAY_BETWEEN_PAGES = 0.3 +DELAY_BETWEEN_OUTCODES = 0.5 MAX_RETRIES = 3 RETRY_BASE_DELAY = 2.0 GRID_CELL_SIZE = 0.01 # degrees for postcode spatial index SEED = 42 +CHECKPOINT_INTERVAL = int(os.environ.get("CHECKPOINT_INTERVAL", "900")) # seconds # Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable. SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3")) diff --git a/finder/openrent.py b/finder/openrent.py index c96dd44..791e79c 100644 --- a/finder/openrent.py +++ b/finder/openrent.py @@ -351,7 +351,7 @@ def parse_search_results(html: str) -> list[dict]:
1 Bed Flat, Location, SW1Y
""" - soup = BeautifulSoup(html, "html.parser") + soup = BeautifulSoup(html, "lxml") properties = [] # Property cards: @@ -486,7 +486,7 @@ def parse_property_detail(html: str) -> dict: - Tables have "Rent PCM", "Deposit", "Bills Included", etc. (NOT bedrooms) - Description in elements with class containing "description" """ - soup = BeautifulSoup(html, "html.parser") + soup = BeautifulSoup(html, "lxml") details: dict = {} # --- Title from h1 --- @@ -624,9 +624,13 @@ def _resolve_outcode_postcodes( pc_coords: dict[str, tuple[float, float]], ) -> list[str]: """Get all postcodes for an outcode from the postcode coordinates lookup.""" + # ONSPD 7-char format: 4-char outcodes have no space before incode + # (e.g., "BH191AB"), while shorter outcodes do (e.g., "E14 5AB"). prefix = outcode + " " - # Also try without space for non-standard format (e.g. "SW1Y" matches "SW1Y 4AA") - return [pcd for pcd in pc_coords if pcd.startswith(prefix)] + results = [pcd for pcd in pc_coords if pcd.startswith(prefix)] + if not results and len(outcode) >= 4: + results = [pcd for pcd in pc_coords if pcd.startswith(outcode) and len(pcd) > len(outcode)] + return results def transform_property( @@ -810,7 +814,7 @@ def search_outcode( if detail_html: detail_data = parse_property_detail(detail_html) # Shorter delay for detail pages (within same outcode) - time.sleep(DELAY_BETWEEN_PAGES * 0.5) + time.sleep(0.15) transformed = transform_property( search_data, diff --git a/finder/pyproject.toml b/finder/pyproject.toml index 05379b6..64be8ed 100644 --- a/finder/pyproject.toml +++ b/finder/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "fake-useragent>=2.2.0", "prometheus-client", "beautifulsoup4", + "lxml", "playwright>=1.58.0", "playwright-stealth>=2.0.2", "camoufox>=0.4.11", diff --git a/finder/scraper.py b/finder/scraper.py index 88c3dd2..4f81aee 100644 --- a/finder/scraper.py +++ b/finder/scraper.py @@ -1,3 +1,4 @@ +import json import logging import random import threading @@ -11,6 +12,7 @@ import httpx from constants import ( ARCGIS_PATH, CHANNELS, + CHECKPOINT_INTERVAL, DATA_DIR, DELAY_BETWEEN_OUTCODES, RELOAD_URL, @@ -233,6 +235,135 @@ def _merge_channel( return all_properties, counts, total_dedup +# --------------------------------------------------------------------------- +# Checkpointing — save/resume partial results across crashes +# --------------------------------------------------------------------------- + + +def _checkpoint_meta_path(): + return DATA_DIR / "checkpoint.json" + + +def _checkpoint_results_path(source: str, channel: str): + return DATA_DIR / f"checkpoint_{source}_{channel}.json" + + +def _save_checkpoint( + shuffled: list[str], + progress: _Progress, + source_results: dict[str, dict[str, list]], + active_sources: list[str], +) -> None: + """Save per-source progress indices and partial results to disk. + + Writes atomically (temp + rename) so a crash mid-write leaves the previous + checkpoint intact. + """ + snap = progress.snapshot() + + meta = { + "seed": SEED, + "num_outcodes": len(shuffled), + "sources": {s: snap.get(s, 0) for s in active_sources}, + "timestamp": time.time(), + } + + # Write result files per source per channel + for source in active_sources: + results = source_results.get(source, {}) + for ch_key in ("BUY", "RENT"): + props = results.get(ch_key, []) + path = _checkpoint_results_path(source, ch_key.lower()) + tmp = path.with_suffix(".tmp") + try: + with open(tmp, "w") as f: + json.dump(props, f, default=str) + tmp.rename(path) + except Exception as e: + log.warning("Failed to write checkpoint %s: %s", path.name, e) + + # Write metadata atomically + tmp = _checkpoint_meta_path().with_suffix(".tmp") + try: + with open(tmp, "w") as f: + json.dump(meta, f) + tmp.rename(_checkpoint_meta_path()) + except Exception as e: + log.warning("Failed to write checkpoint metadata: %s", e) + return + + total = sum(len(source_results.get(s, {}).get(ch, [])) + for s in active_sources for ch in ("BUY", "RENT")) + log.info( + "Checkpoint saved: %s (%d properties)", + {s: snap.get(s, 0) for s in active_sources}, + total, + ) + + +def _load_checkpoint( + shuffled: list[str], +) -> tuple[dict[str, int], dict[str, dict[str, list]]] | None: + """Load checkpoint if it exists and matches the current outcode list. + + Returns (start_indices, loaded_results) or None if no valid checkpoint. + """ + path = _checkpoint_meta_path() + if not path.exists(): + return None + + try: + with open(path) as f: + meta = json.load(f) + except Exception: + log.warning("Checkpoint file corrupt, starting fresh") + _clear_checkpoint() + return None + + if meta.get("seed") != SEED or meta.get("num_outcodes") != len(shuffled): + log.info("Checkpoint from different run configuration, discarding") + _clear_checkpoint() + return None + + start_indices: dict[str, int] = {} + loaded_results: dict[str, dict[str, list]] = {} + + for source, completed in meta.get("sources", {}).items(): + start_indices[source] = completed + loaded_results[source] = {"BUY": [], "RENT": []} + for channel in ("buy", "rent"): + rpath = _checkpoint_results_path(source, channel) + if rpath.exists(): + try: + with open(rpath) as f: + loaded_results[source][channel.upper()] = json.load(f) + except Exception: + log.warning( + "Checkpoint results for %s/%s corrupt, restarting %s", + source, channel, source, + ) + start_indices[source] = 0 + loaded_results[source] = {"BUY": [], "RENT": []} + break + + elapsed_since = time.time() - meta.get("timestamp", 0) + log.info( + "Resuming from checkpoint (saved %.0fm ago): %s", + elapsed_since / 60, + start_indices, + ) + return start_indices, loaded_results + + +def _clear_checkpoint() -> None: + """Remove all checkpoint files after successful completion.""" + for path in DATA_DIR.glob("checkpoint*"): + try: + path.unlink() + except Exception: + pass + + def run_scrape( outcodes: list[str], pc_index: PostcodeSpatialIndex, @@ -293,15 +424,40 @@ def run_scrape( progress = _Progress() + # --- Resume from checkpoint if available --- + start_indices: dict[str, int] = {} + checkpoint = _load_checkpoint(shuffled) + if checkpoint: + start_indices, loaded = checkpoint + source_to_results = {"rm": rm_results, "hk": hk_results, "or": or_results, "zp": zp_results} + for src, data in loaded.items(): + if src in source_to_results: + for ch in ("BUY", "RENT"): + source_to_results[src][ch] = data.get(ch, []) + # Reassign in case references changed + rm_results = source_to_results["rm"] + hk_results = source_to_results["hk"] + or_results = source_to_results["or"] + zp_results = source_to_results["zp"] + # Pre-set progress for resumed sources + for src, idx in start_indices.items(): + if idx > 0: + progress.update(src, idx) + # --- Source worker closures --- # Each worker owns its client lifecycle and iterates all outcodes for both # channels. On auth failure, it refreshes cookies and continues. On fatal # failure, it marks itself as done and returns partial results. def rm_worker(): + rm_start = start_indices.get("rm", 0) + if rm_start > 0: + log.info("Rightmove resuming from outcode %d/%d", rm_start, len(shuffled)) client = make_client() try: for i, outcode in enumerate(shuffled): + if i < rm_start: + continue try: outcode_id = resolve_outcode_id(client, outcode) except Exception as e: @@ -344,11 +500,16 @@ def run_scrape( homecouk_enabled.set(0) progress.update("hk", len(shuffled)) return + hk_start = start_indices.get("hk", 0) + if hk_start > 0: + log.info("home.co.uk resuming from outcode %d/%d", hk_start, len(shuffled)) client = make_homecouk_client(*hk_result) log.info("home.co.uk scraping ENABLED") homecouk_enabled.set(1) try: for i, outcode in enumerate(shuffled): + if i < hk_start: + continue for ch_cfg in CHANNELS: ch = ch_cfg["channel"] try: @@ -403,11 +564,16 @@ def run_scrape( openrent_enabled.set(0) progress.update("or", len(shuffled)) return + or_start = start_indices.get("or", 0) + if or_start > 0: + log.info("OpenRent resuming from outcode %d/%d", or_start, len(shuffled)) client = make_openrent_client(*or_result) log.info("OpenRent scraping ENABLED") openrent_enabled.set(1) try: for i, outcode in enumerate(shuffled): + if i < or_start: + continue # OpenRent is RENT-only try: props = openrent_search_outcode( @@ -470,14 +636,31 @@ def run_scrape( progress.update("zp", len(shuffled)) return + zp_start = start_indices.get("zp", 0) + if zp_start > 0: + log.info("Zoopla resuming from outcode %d/%d", zp_start, len(shuffled)) + try: for i, outcode in enumerate(shuffled): + if i < zp_start: + continue + search_url = None for ch_cfg in CHANNELS: ch = ch_cfg["channel"] + # Build direct URL for second channel by swapping path + direct_url = None + if search_url: + if ch == "BUY": + direct_url = search_url.replace("/to-rent/", "/for-sale/") + else: + direct_url = search_url.replace("/for-sale/", "/to-rent/") try: - props = zoopla_search_outcode( - page, outcode, ch, pc_index, pc_coords + props, result_url = zoopla_search_outcode( + page, outcode, ch, pc_index, pc_coords, + base_search_url=direct_url, ) + if result_url: + search_url = result_url zp_results[ch].extend(props) if props: log.info("Zoopla %s: +%d properties", outcode, len(props)) @@ -548,8 +731,15 @@ def run_scrape( # --- Monitor progress while workers run --- + # Map source names to result dicts for checkpointing + source_results_map = { + "rm": rm_results, "hk": hk_results, + "or": or_results, "zp": zp_results, + } + scrape_start = time.time() last_log = 0.0 + last_checkpoint = time.time() try: while any(t.is_alive() for t in threads): @@ -577,8 +767,9 @@ def run_scrape( status.zp_properties = len(zp_results["BUY"]) + len(zp_results["RENT"]) _sync_gauges() - # Log progress every 30 seconds now = time.time() + + # Log progress every 30 seconds if now - last_log >= 30: elapsed = now - scrape_start per_source = ", ".join( @@ -595,10 +786,26 @@ def run_scrape( ) last_log = now + # Save checkpoint periodically + if now - last_checkpoint >= CHECKPOINT_INTERVAL: + try: + _save_checkpoint( + shuffled, progress, source_results_map, active_sources, + ) + except Exception as e: + log.warning("Checkpoint save failed: %s", e) + last_checkpoint = now + time.sleep(5) except Exception as e: log.exception("Monitor loop error: %s", e) + # Save final checkpoint before joining (in case merge/write fails) + try: + _save_checkpoint(shuffled, progress, source_results_map, active_sources) + except Exception: + pass + for t in threads: t.join() @@ -645,6 +852,9 @@ def run_scrape( total_dedup, ) + # Scrape completed successfully — clear checkpoint + _clear_checkpoint() + with status_lock: status.state = "done" status.finished_at = time.time() diff --git a/finder/zoopla.py b/finder/zoopla.py index 052794f..59372ad 100644 --- a/finder/zoopla.py +++ b/finder/zoopla.py @@ -263,6 +263,46 @@ def _ensure_not_challenged(page) -> None: # --------------------------------------------------------------------------- +def _navigate_direct(page, url: str) -> bool: + """Navigate directly to a Zoopla search URL (skipping the homepage flow). + + Used to load the second channel (e.g., RENT after BUY) for the same outcode + by swapping the path component. Falls back gracefully — returns False if + the page has no listings, so the caller can retry via the full search flow. + """ + try: + page.goto(url, wait_until="domcontentloaded", timeout=30000) + except Exception as e: + log.debug("Direct navigation failed: %s", e) + return False + _ensure_not_challenged(page) + + # Wait for listing content to hydrate + try: + page.wait_for_function( + """() => { + const cards = document.querySelectorAll( + '[data-testid="regular-listings"] > div' + ); + if (cards.length === 0) return false; + for (const card of cards) { + const t = card.innerText || ''; + if (t.includes('\\u00a3') && t.length > 50) return true; + } + return false; + }""", + timeout=8000, + ) + except Exception: + # Check if the page has any listings at all + has_listings = page.query_selector('a[href*="/details/"]') + if not has_listings: + return False + time.sleep(1.5) + + return True + + def _navigate_search(page, outcode: str, channel: str) -> bool: """Navigate to search results for an outcode via the homepage search flow. @@ -270,12 +310,12 @@ def _navigate_search(page, outcode: str, channel: str) -> bool: Raises TurnstileError if Cloudflare blocks us.""" # Navigate to homepage to reset search state page.goto(f"{ZOOPLA_BASE}/", wait_until="domcontentloaded", timeout=30000) - time.sleep(2) + time.sleep(0.5) _ensure_not_challenged(page) # Dismiss cookie consent (may reappear after navigation) page.evaluate(_DISMISS_COOKIES_JS) - time.sleep(1) + time.sleep(0.3) # Select Buy/Rent tab if channel == "RENT": @@ -284,7 +324,7 @@ def _navigate_search(page, outcode: str, channel: str) -> bool: ) if rent_tab: rent_tab.click() - time.sleep(0.5) + time.sleep(0.2) # Find and fill search input search_input = page.query_selector( @@ -295,10 +335,10 @@ def _navigate_search(page, outcode: str, channel: str) -> bool: return False search_input.click() - time.sleep(0.3) + time.sleep(0.1) search_input.fill("") search_input.type(outcode, delay=60) - time.sleep(2) + time.sleep(1.2) # Select first autocomplete suggestion first_option = page.query_selector('[role="option"]') @@ -307,7 +347,7 @@ def _navigate_search(page, outcode: str, channel: str) -> bool: return False first_option.click() - time.sleep(0.5) + time.sleep(0.2) # Click search button search_btn = page.query_selector('button:has-text("Search")') @@ -326,6 +366,29 @@ def _navigate_search(page, outcode: str, channel: str) -> bool: time.sleep(4) _ensure_not_challenged(page) + # Wait for client-side hydration to populate listing content (prices, addresses). + # The structural container appears in server-rendered HTML before React hydrates + # the actual card content — extracting too early yields empty price/address fields. + try: + page.wait_for_function( + """() => { + const cards = document.querySelectorAll( + '[data-testid="regular-listings"] > div' + ); + if (cards.length === 0) return false; + for (const card of cards) { + const t = card.innerText || ''; + if (t.includes('\\u00a3') && t.length > 50) return true; + } + return false; + }""", + timeout=8000, + ) + except Exception: + # Content never appeared — extraction will likely fail but let it try + log.debug("Listing content hydration wait timed out — prices may not have rendered") + time.sleep(2) + return True @@ -437,8 +500,25 @@ def _paginate(page, total_results: int, channel: str) -> list[dict]: try: page.goto(next_url, wait_until="domcontentloaded", timeout=30000) - time.sleep(4) _ensure_not_challenged(page) + # Wait for listing content instead of fixed sleep + try: + page.wait_for_function( + """() => { + const cards = document.querySelectorAll( + '[data-testid="regular-listings"] > div' + ); + if (cards.length === 0) return false; + for (const card of cards) { + const t = card.innerText || ''; + if (t.includes('\\u00a3') && t.length > 50) return true; + } + return false; + }""", + timeout=8000, + ) + except Exception: + time.sleep(1.5) except TurnstileError: raise except Exception as e: @@ -546,9 +626,16 @@ def transform_property( # Try outcode-level fallback outcode = _extract_outcode(address) if outcode: + # ONSPD 7-char format: 4-char outcodes have no space before incode + # (e.g., "BH191AB"), while shorter outcodes do (e.g., "E14 5AB"). + # Check both formats to handle all outcode lengths. prefix = outcode + " " for pcd, coords in pc_coords.items(): - if pcd.startswith(prefix): + if pcd.startswith(prefix) or ( + len(outcode) >= 4 + and pcd.startswith(outcode) + and len(pcd) > len(outcode) + ): postcode = pcd lat, lng = coords break @@ -608,17 +695,32 @@ def search_outcode( channel: str, pc_index: PostcodeSpatialIndex, pc_coords: dict[str, tuple[float, float]], -) -> list[dict]: + base_search_url: str | None = None, +) -> tuple[list[dict], str | None]: """Search Zoopla for properties in one outcode. Takes a live Camoufox Page (from launch_browser). Navigates through the search flow, extracts listings from rendered DOM, and transforms to the standard output schema. + If base_search_url is provided (from a previous channel search for the same + outcode), tries direct URL navigation first — skipping the slow homepage + search flow. Falls back to full navigation if direct fails. + + Returns (properties, search_url) where search_url can be passed to the next + channel call for this outcode. + Raises TurnstileError if Cloudflare blocks us mid-session. """ - if not _navigate_search(page, outcode, channel): - return [] + navigated = False + if base_search_url: + navigated = _navigate_direct(page, base_search_url) + if navigated: + log.debug("Zoopla %s %s: used direct URL navigation", outcode, channel) + + if not navigated: + if not _navigate_search(page, outcode, channel): + return [], None total_results = _get_result_count(page) @@ -632,7 +734,7 @@ def search_outcode( "DOM selectors may need updating", outcode, channel, total_results, ) - return [] + return [], None channel_label = "buy" if channel == "BUY" else "rent" properties = [] @@ -646,10 +748,13 @@ def search_outcode( dropped += 1 if dropped and not properties: + # Log a sample raw listing to diagnose which fields are missing + sample = raw_listings[0] if raw_listings else {} log.debug( "Zoopla %s %s: extracted %d raw listings but all %d dropped in transform " - "(no price/postcode/coords)", + "(no price/postcode/coords). Sample raw: price=%s address=%r", outcode, channel, len(raw_listings), dropped, + sample.get("price"), sample.get("address", ""), ) elif dropped > len(raw_listings) // 2: log.debug( @@ -657,4 +762,4 @@ def search_outcode( outcode, channel, dropped, len(raw_listings), ) - return properties + return properties, page.url diff --git a/frontend/src/components/map/FeatureBrowser.tsx b/frontend/src/components/map/FeatureBrowser.tsx index defba35..adba0be 100644 --- a/frontend/src/components/map/FeatureBrowser.tsx +++ b/frontend/src/components/map/FeatureBrowser.tsx @@ -130,12 +130,7 @@ export default function FeatureBrowser({ className="flex items-center justify-between px-3 py-1.5 hover:bg-teal-50 dark:hover:bg-teal-900/30 dark:text-warm-300" >
- - {f.description && ( - - {f.description} - - )} +
setTravelInfoMode(mode)} title="Feature info" + size="md" > - + diff --git a/frontend/src/components/map/Filters.tsx b/frontend/src/components/map/Filters.tsx index 7aec19a..39015e6 100644 --- a/frontend/src/components/map/Filters.tsx +++ b/frontend/src/components/map/Filters.tsx @@ -112,8 +112,8 @@ function SliderLabels({ onValueChange?: (v: [number, number]) => void; }) { const range = max - min || 1; - const leftPct = ((value[0] - min) / range) * 100; - const rightPct = ((value[1] - min) / range) * 100; + const leftPct = Math.max(0, Math.min(100, ((value[0] - min) / range) * 100)); + const rightPct = Math.max(0, Math.min(100, ((value[1] - min) / range) * 100)); const labels = displayValues || value; const minLabel = isAtMin ? 'min' : formatFilterValue(labels[0], raw); @@ -133,7 +133,7 @@ function SliderLabels({ onValueChange([Math.min(v, labels[1]), labels[1]])} + onCommit={(v) => onValueChange([v, Math.max(v, labels[1])])} prefix={feature.prefix} suffix={feature.suffix} style={{ left: `${leftPct}%`, transform: leftTranslate }} @@ -141,7 +141,7 @@ function SliderLabels({ onValueChange([labels[0], Math.max(v, labels[0])])} + onCommit={(v) => onValueChange([Math.min(labels[0], v), v])} prefix={feature.prefix} suffix={feature.suffix} style={{ left: `${rightPct}%`, transform: rightTranslate }} @@ -184,6 +184,7 @@ interface FiltersProps { onTravelTimeRemoveEntry: (index: number) => void; onTravelTimeSetDestination: (index: number, slug: string, label: string) => void; onTravelTimeRangeChange: (index: number, range: [number, number]) => void; + onTravelTimeDragEnd: (index: number) => void; onTravelTimeToggleBest: (index: number) => void; aiFilterLoading: boolean; aiFilterError: string | null; @@ -220,6 +221,7 @@ export default memo(function Filters({ onTravelTimeRemoveEntry, onTravelTimeSetDestination, onTravelTimeRangeChange, + onTravelTimeDragEnd, onTravelTimeToggleBest, aiFilterLoading, aiFilterError, @@ -415,7 +417,7 @@ export default memo(function Filters({ -
+
onTogglePin(travelFieldKey(entry))} onSetDestination={(slug, label) => onTravelTimeSetDestination(index, slug, label)} onTimeRangeChange={(range) => onTravelTimeRangeChange(index, range)} + onDragStart={() => onDragStart(travelFieldKey(entry))} + onDragChange={onDragChange} + onDragEnd={() => onTravelTimeDragEnd(index)} onToggleBest={() => onTravelTimeToggleBest(index)} onRemove={() => onTravelTimeRemoveEntry(index)} /> @@ -592,7 +599,7 @@ export default memo(function Filters({ min={scale ? 0 : feature.min!} max={scale ? 100 : feature.max!} value={sliderValue} - displayValues={scale ? displayValue : undefined} + displayValues={displayValue} isAtMin={isAtMin} isAtMax={isAtMax} raw={feature.raw} diff --git a/frontend/src/components/map/Map.tsx b/frontend/src/components/map/Map.tsx index 4519b16..001dfc7 100644 --- a/frontend/src/components/map/Map.tsx +++ b/frontend/src/components/map/Map.tsx @@ -1,5 +1,5 @@ import { useCallback, useRef, useEffect, useState, useMemo, memo } from 'react'; -import { Map as MapGL, useControl } from 'react-map-gl/maplibre'; +import { Map as MapGL, useControl, ScaleControl } from 'react-map-gl/maplibre'; import { MapboxOverlay } from '@deck.gl/mapbox'; import 'maplibre-gl/dist/maplibre-gl.css'; import type { @@ -212,6 +212,9 @@ export default memo(function Map({ maxBounds={MAP_BOUNDS} > + {!screenshotMode && ( + + )} {screenshotMode ? ( ogMode ? ( diff --git a/frontend/src/components/map/MapPage.tsx b/frontend/src/components/map/MapPage.tsx index 62b93b7..a2976ee 100644 --- a/frontend/src/components/map/MapPage.tsx +++ b/frontend/src/components/map/MapPage.tsx @@ -105,8 +105,8 @@ export default function MapPage({ const [selectedPOICategories, setSelectedPOICategories] = useState>(initialPOICategories); - const [leftPaneWidth, leftPaneHandlers] = usePaneResize(384, 200, 600, 'left'); - const [rightPaneWidth, rightPaneHandlers] = usePaneResize(384, 200, 500, 'right'); + const [leftPaneWidth, leftPaneHandlers] = usePaneResize(384, 200, 0.45, 'left'); + const [rightPaneWidth, rightPaneHandlers] = usePaneResize(384, 200, 0.45, 'right'); const [mobileDrawerOpen, setMobileDrawerOpen] = useState(false); const [poiPaneOpen, setPoiPaneOpen] = useState(false); @@ -141,6 +141,7 @@ export default function MapPage({ handleDragStart, handleDragChange, handleDragEnd, + handleDragEndNoCommit, handleTogglePin, handleSetPin, handleCancelPin, @@ -204,12 +205,8 @@ export default function MapPage({ const handleTravelTimeSetDestination = useCallback( (index: number, slug: string, label: string) => { travelTime.handleSetDestination(index, slug, label); - const entry = travelTime.entries[index]; - if (entry) { - handleSetPin(`tt_${entry.mode}_${slug}`); - } }, - [travelTime.handleSetDestination, travelTime.entries, handleSetPin] + [travelTime.handleSetDestination] ); const handleTravelTimeRemoveEntry = useCallback( @@ -223,6 +220,14 @@ export default function MapPage({ [travelTime.handleRemoveEntry, travelTime.entries, pinnedFeature, handleCancelPin] ); + const handleTravelTimeDragEnd = useCallback( + (index: number) => { + const dv = handleDragEndNoCommit(); + if (dv) travelTime.handleTimeRangeChange(index, dv); + }, + [handleDragEndNoCommit, travelTime.handleTimeRangeChange] + ); + const license = useLicense(); const mapFlyToRef = useRef<((lat: number, lng: number, zoom: number) => void) | null>(null); @@ -571,6 +576,7 @@ export default function MapPage({ onTravelTimeRemoveEntry={handleTravelTimeRemoveEntry} onTravelTimeSetDestination={handleTravelTimeSetDestination} onTravelTimeRangeChange={travelTime.handleTimeRangeChange} + onTravelTimeDragEnd={handleTravelTimeDragEnd} onTravelTimeToggleBest={travelTime.handleToggleBest} aiFilterLoading={aiFilters.loading} aiFilterError={aiFilters.error} diff --git a/frontend/src/components/map/TravelTimeCard.tsx b/frontend/src/components/map/TravelTimeCard.tsx index 02b68ba..e432a65 100644 --- a/frontend/src/components/map/TravelTimeCard.tsx +++ b/frontend/src/components/map/TravelTimeCard.tsx @@ -19,9 +19,14 @@ interface TravelTimeCardProps { timeRange: [number, number] | null; useBest: boolean; isPinned: boolean; + isActive: boolean; + dragValue: [number, number] | null; onTogglePin: () => void; onSetDestination: (slug: string, label: string) => void; onTimeRangeChange: (range: [number, number]) => void; + onDragStart: () => void; + onDragChange: (value: [number, number]) => void; + onDragEnd: () => void; onToggleBest: () => void; onRemove: () => void; } @@ -33,9 +38,14 @@ export function TravelTimeCard({ timeRange, useBest, isPinned, + isActive, + dragValue, onTogglePin, onSetDestination, onTimeRangeChange, + onDragStart, + onDragChange, + onDragEnd, onToggleBest, onRemove, }: TravelTimeCardProps) { @@ -52,13 +62,13 @@ export function TravelTimeCard({ const sliderMin = 0; const sliderMax = 120; - const displayRange = timeRange ?? [sliderMin, sliderMax]; + const displayRange = isActive && dragValue ? dragValue : (timeRange ?? [sliderMin, sliderMax]); const ModeIcon = MODE_ICONS[mode]; return (
{/* Header */}
@@ -130,7 +140,9 @@ export function TravelTimeCard({ max={sliderMax} step={1} value={[displayRange[0], displayRange[1]]} - onValueChange={([min, max]) => onTimeRangeChange([min, max])} + onValueChange={([min, max]) => onDragChange([min, max])} + onPointerDown={() => onDragStart()} + onPointerUp={() => onDragEnd()} />
{formatFilterValue(displayRange[0])} min diff --git a/frontend/src/components/ui/FeatureIcons.tsx b/frontend/src/components/ui/FeatureIcons.tsx index 1dbce65..f07fbb0 100644 --- a/frontend/src/components/ui/FeatureIcons.tsx +++ b/frontend/src/components/ui/FeatureIcons.tsx @@ -23,7 +23,7 @@ export function FeatureActions({
{feature.detail && onShowInfo && ( onShowInfo(feature)} title="Feature info" size="md"> - + )} - + {onAdd && ( )} {onRemove && ( diff --git a/frontend/src/components/ui/FeatureLabel.tsx b/frontend/src/components/ui/FeatureLabel.tsx index d261859..178fba3 100644 --- a/frontend/src/components/ui/FeatureLabel.tsx +++ b/frontend/src/components/ui/FeatureLabel.tsx @@ -14,6 +14,7 @@ interface FeatureLabelProps { onShowInfo?: (feature: FeatureMeta) => void; className?: string; size?: 'xs' | 'sm'; + description?: string; } export function FeatureLabel({ @@ -21,6 +22,7 @@ export function FeatureLabel({ onShowInfo, className = '', size = 'xs', + description, }: FeatureLabelProps) { const textClass = size === 'sm' ? 'text-sm' : 'text-xs'; const iconClass = 'w-3.5 h-3.5 text-teal-600 dark:text-teal-400 shrink-0'; @@ -31,12 +33,8 @@ export function FeatureLabel({ ? feature.modes.map((m) => MODE_LABELS[m] || m).join(' · ') : null; - return ( -
- {featureIcon} - {GroupIcon && } + const nameContent = ( + <> @@ -56,6 +54,23 @@ export function FeatureLabel({ )} + + ); + + return ( +
+ {featureIcon} + {GroupIcon && } + {description ? ( +
+
{nameContent}
+ {description} +
+ ) : ( + nameContent + )}
); } diff --git a/frontend/src/hooks/useDeckLayers.ts b/frontend/src/hooks/useDeckLayers.ts index 920af1e..438d240 100644 --- a/frontend/src/hooks/useDeckLayers.ts +++ b/frontend/src/hooks/useDeckLayers.ts @@ -24,7 +24,7 @@ import { POI_CLUSTER_MAX_ZOOM, } from '../lib/consts'; import { emojiToTwemojiUrl, getFeatureFillColor } from '../lib/map-utils'; -import { type TravelTimeEntry, travelFieldKey } from './useTravelTime'; +import type { TravelTimeEntry } from './useTravelTime'; import { MarchingAntsExtension } from '../lib/MarchingAntsExtension'; interface UseDeckLayersProps { @@ -120,9 +120,6 @@ export function useDeckLayers({ const hoveredPostcodeRef = useRef(hoveredPostcode); hoveredPostcodeRef.current = hoveredPostcode; - const travelTimeEntriesRef = useRef(travelTimeEntries); - travelTimeEntriesRef.current = travelTimeEntries; - const colorFeatureMeta = useMemo( () => (viewFeature ? features.find((f) => f.name === viewFeature) || null : null), [viewFeature, features] @@ -302,28 +299,6 @@ export function useDeckLayers({ getHexagon: (d) => d.h3, getFillColor: (d) => { const dark = isDarkRef.current; - const entries = travelTimeEntriesRef.current; - - // Dim-filter: all travel entries with timeRange dim hexagons outside range - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - if (!entry.timeRange || !entry.slug) continue; - const fk = travelFieldKey(entry); - const modeVal = d[`avg_${fk}`]; - if ( - modeVal == null || - (modeVal as number) < entry.timeRange[0] || - (modeVal as number) > entry.timeRange[1] - ) { - return (dark ? [60, 55, 50, 60] : [180, 180, 180, 60]) as [ - number, - number, - number, - number, - ]; - } - } - const vf = viewFeatureRef.current; const clr = colorRangeRef.current; const fr = filterRangeRef.current; @@ -425,28 +400,6 @@ export function useDeckLayers({ getFillColor: (f) => { const d = f.properties; const dark = isDarkRef.current; - const entries = travelTimeEntriesRef.current; - - // Dim-filter: all travel entries with timeRange dim postcodes outside range - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - if (!entry.timeRange || !entry.slug) continue; - const fk = travelFieldKey(entry); - const modeVal = d[`avg_${fk}`]; - if ( - modeVal == null || - (modeVal as number) < entry.timeRange[0] || - (modeVal as number) > entry.timeRange[1] - ) { - return (dark ? [60, 55, 50, 60] : [180, 180, 180, 60]) as [ - number, - number, - number, - number, - ]; - } - } - const vf = viewFeatureRef.current; const clr = colorRangeRef.current; const fr = filterRangeRef.current; diff --git a/frontend/src/hooks/useFilters.ts b/frontend/src/hooks/useFilters.ts index 73e27f3..3235d11 100644 --- a/frontend/src/hooks/useFilters.ts +++ b/frontend/src/hooks/useFilters.ts @@ -120,6 +120,20 @@ export function useFilters({ initialFilters, features }: UseFiltersOptions) { dragValueRef.current = null; }, []); + /** End drag without committing to filters — caller handles the commit (e.g. travel time). */ + const handleDragEndNoCommit = useCallback((): [number, number] | null => { + if (pendingDragRef.current) { + pendingDragRef.current = null; + return null; + } + const dv = dragValueRef.current; + setActiveFeature(null); + setDragValue(null); + dragActiveRef.current = null; + dragValueRef.current = null; + return dv; + }, []); + const handleSetFilters = useCallback((newFilters: FeatureFilters) => { setFilters(newFilters); setActiveFeature(null); @@ -159,6 +173,7 @@ export function useFilters({ initialFilters, features }: UseFiltersOptions) { handleDragStart, handleDragChange, handleDragEnd, + handleDragEndNoCommit, handleTogglePin, handleSetPin, handleCancelPin, diff --git a/frontend/src/hooks/useMapData.ts b/frontend/src/hooks/useMapData.ts index 661e7a1..9ef6d16 100644 --- a/frontend/src/hooks/useMapData.ts +++ b/frontend/src/hooks/useMapData.ts @@ -81,25 +81,34 @@ export function useMapData({ ); // Build the travel param string from entries with destinations. - // timeRange is NOT included — range filtering is handled purely client-side - // (dimming in useDeckLayers) so slider changes never trigger server refetches. - const travelParam = useMemo((): string => { - const segments: string[] = []; - for (const entry of travelTimeEntries) { - if (!entry.slug) continue; - let seg = `${entry.mode}:${entry.slug}`; - if (entry.useBest) seg += ':best'; - segments.push(seg); - } - return segments.join('|'); - }, [travelTimeEntries]); + // Format: mode:slug[:best][:min:max] — server filters rows outside [min,max]. + // When excludeFieldKey is set, that entry's time range is omitted (for drag preview). + const buildTravelParam = useCallback( + (excludeFieldKey?: string): string => { + const segments: string[] = []; + for (const entry of travelTimeEntries) { + if (!entry.slug) continue; + let seg = `${entry.mode}:${entry.slug}`; + if (entry.useBest) seg += ':best'; + const isExcluded = excludeFieldKey === `tt_${entry.mode}_${entry.slug}`; + if (entry.timeRange && !isExcluded) seg += `:${entry.timeRange[0]}:${entry.timeRange[1]}`; + segments.push(seg); + } + return segments.join('|'); + }, + [travelTimeEntries] + ); + + const travelParam = useMemo(() => buildTravelParam(), [buildTravelParam]); // Keep activeFeatureRef in sync useEffect(() => { activeFeatureRef.current = activeFeature; }, [activeFeature]); - // Drag prefetch: when activeFeature starts, fetch data excluding that filter + // Drag prefetch: when activeFeature starts, fetch data excluding that filter. + // For regular filters: excludes the filter from the filter string. + // For travel time: excludes the time range from that entry's travel param segment. useEffect(() => { if (!activeFeature || !bounds) return; @@ -108,11 +117,14 @@ export function useMapData({ const filtersStr = buildFilterString(filters, features, activeFeature); const boundsStr = `${bounds.south},${bounds.west},${bounds.north},${bounds.east}`; + const isTravelTimeDrag = activeFeature.startsWith('tt_'); + const dragTravelParam = isTravelTimeDrag ? buildTravelParam(activeFeature) : travelParam; if (usePostcodeView) { const params = new URLSearchParams({ bounds: boundsStr }); if (filtersStr) params.set('filters', filtersStr); params.set('fields', activeFeature); + if (dragTravelParam) params.set('travel', dragTravelParam); fetch(apiUrl('postcodes', params), authHeaders({ signal: dragAbortRef.current.signal })) .then((res) => res.json()) @@ -129,7 +141,7 @@ export function useMapData({ }); if (filtersStr) params.set('filters', filtersStr); params.set('fields', activeFeature); - if (travelParam) params.set('travel', travelParam); + if (dragTravelParam) params.set('travel', dragTravelParam); fetch(apiUrl('hexagons', params), authHeaders({ signal: dragAbortRef.current.signal })) .then((res) => res.json()) @@ -147,7 +159,7 @@ export function useMapData({ dragAbortRef.current = null; } }; - }, [activeFeature, bounds, resolution, filters, features, usePostcodeView, travelParam]); + }, [activeFeature, bounds, resolution, filters, features, usePostcodeView, travelParam, buildTravelParam]); // Fetch hexagons or postcodes when bounds/filters change useEffect(() => { diff --git a/frontend/src/hooks/usePaneResize.ts b/frontend/src/hooks/usePaneResize.ts index abd04e7..57ab46f 100644 --- a/frontend/src/hooks/usePaneResize.ts +++ b/frontend/src/hooks/usePaneResize.ts @@ -24,10 +24,11 @@ export function usePaneResize( const handlePointerMove = useCallback( (e: React.PointerEvent) => { if (!draggingRef.current) return; + const resolvedMax = maxWidth <= 1 ? window.innerWidth * maxWidth : maxWidth; const newWidth = side === 'left' - ? Math.min(maxWidth, Math.max(minWidth, e.clientX)) - : Math.min(maxWidth, Math.max(minWidth, window.innerWidth - e.clientX)); + ? Math.min(resolvedMax, Math.max(minWidth, e.clientX)) + : Math.min(resolvedMax, Math.max(minWidth, window.innerWidth - e.clientX)); setWidth(newWidth); }, [side, minWidth, maxWidth] diff --git a/frontend/src/index.css b/frontend/src/index.css index c4b3b10..28a675e 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -201,6 +201,13 @@ h3 { } } +/* MapLibre scale control — dark mode */ +.dark .maplibregl-ctrl-scale { + border-color: #d6d3d1; + color: #d6d3d1; + background-color: rgba(28, 25, 23, 0.5); +} + /* Hide scrollbar for pill groups on mobile */ .scrollbar-hide { -ms-overflow-style: none; diff --git a/frontend/src/lib/map-utils.ts b/frontend/src/lib/map-utils.ts index 1829b39..805a717 100644 --- a/frontend/src/lib/map-utils.ts +++ b/frontend/src/lib/map-utils.ts @@ -10,7 +10,6 @@ import { BUFFER_MULTIPLIER, ENUM_PALETTE, } from './consts'; - const ROAD_OPACITY = 0.4; export function getMapStyle(theme: 'light' | 'dark'): StyleSpecification {