Add OpenRent

finder/openrent.py

@@ -0,0 +1,612 @@
+"""OpenRent (openrent.co.uk) scraper — rental properties only.
+
+OpenRent is behind AWS WAF, so we use FlareSolverr to solve the challenge
+and get valid cookies. Then we use curl_cffi with Chrome TLS impersonation
+to make requests with those cookies (same pattern as homecouk.py).
+
+OpenRent is a rental-only platform, so this scraper only handles RENT channel.
+
+HTML parsing notes:
+  OpenRent server-renders property cards in the search results page.
+  Property cards use class "pli" (property list item). Each card contains
+  a link to the property detail page, price, bedrooms, and address info.
+  The CSS selectors below may need updating if OpenRent changes their markup.
+"""
+
+import logging
+import os
+import re
+import time
+
+import httpx
+from bs4 import BeautifulSoup
+from curl_cffi.requests import Session
+from curl_cffi.requests.errors import RequestsError
+
+from constants import (
+    DELAY_BETWEEN_PAGES,
+    OPENRENT_BASE,
+    PROPERTY_TYPE_MAP,
+    RETRY_BASE_DELAY,
+)
+from metrics import (
+    flaresolverr_attempts_total,
+    openrent_errors_total,
+    openrent_properties_scraped,
+    openrent_requests_total,
+)
+from spatial import PostcodeSpatialIndex
+
+log = logging.getLogger("openrent")
+
+
+class WafChallengeError(Exception):
+    """Raised when OpenRent returns a WAF challenge, indicating cookies need refresh."""
+
+
+FLARESOLVERR_URL = os.environ.get("FLARESOLVERR_URL", "http://flaresolverr:8191")
+
+
+# ---------------------------------------------------------------------------
+# Cookie / session management (mirrors homecouk.py pattern)
+# ---------------------------------------------------------------------------
+
+
+def solve_waf() -> tuple[dict[str, str], str] | None:
+    """Use FlareSolverr to solve the AWS WAF challenge.
+    Returns (cookies_dict, user_agent) or None on failure."""
+    log.info("Solving AWS WAF challenge via FlareSolverr at %s", FLARESOLVERR_URL)
+    try:
+        with httpx.Client(timeout=120) as client:
+            resp = client.post(
+                f"{FLARESOLVERR_URL}/v1",
+                json={
+                    "cmd": "request.get",
+                    "url": f"{OPENRENT_BASE}/properties-to-rent/?term=london&isLive=true",
+                    "maxTimeout": 60000,
+                },
+            )
+            if resp.status_code != 200:
+                log.error("FlareSolverr returned HTTP %d", resp.status_code)
+                return None
+
+            data = resp.json()
+            if data.get("status") != "ok":
+                log.error("FlareSolverr error: %s", data.get("message", "unknown"))
+                return None
+
+            solution = data["solution"]
+            raw_cookies = solution.get("cookies", [])
+            user_agent = solution.get("userAgent", "")
+
+            cookies = {}
+            for c in raw_cookies:
+                name = c.get("name", "")
+                if name:
+                    cookies[name] = c["value"]
+
+            if not cookies:
+                log.error("FlareSolverr solved but returned no cookies")
+                flaresolverr_attempts_total.labels(result="no_cookies").inc()
+                return None
+
+            log.info(
+                "AWS WAF solved — got %d cookies, UA: %s",
+                len(cookies), user_agent[:60],
+            )
+            flaresolverr_attempts_total.labels(result="success").inc()
+            return cookies, user_agent
+
+    except (httpx.ConnectError, httpx.ReadTimeout) as e:
+        log.warning("FlareSolverr not available: %s", e)
+        flaresolverr_attempts_total.labels(result="unavailable").inc()
+        return None
+    except Exception as e:
+        log.error("FlareSolverr error: %s", e)
+        flaresolverr_attempts_total.labels(result="error").inc()
+        return None
+
+
+def load_cookies() -> tuple[dict[str, str], str] | None:
+    """Get OpenRent cookies + user-agent.
+    Tries FlareSolverr first, then falls back to environment variables."""
+    result = solve_waf()
+    if result:
+        return result
+
+    # Fall back to env vars
+    waf_token = os.environ.get("OPENRENT_WAF_TOKEN", "")
+    if not waf_token:
+        return None
+
+    user_agent = os.environ.get(
+        "OPENRENT_USER_AGENT",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/145.0.0.0 Safari/537.36",
+    )
+    return {"aws-waf-token": waf_token}, user_agent
+
+
+def make_client(cookies: dict[str, str], user_agent: str) -> Session:
+    """Create a curl_cffi Session configured for OpenRent.
+    Uses Chrome TLS impersonation so AWS WAF cookies remain valid."""
+    session = Session(impersonate="chrome")
+    session.headers.update({
+        "User-Agent": user_agent,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-GB,en;q=0.9",
+    })
+    for name, value in cookies.items():
+        session.cookies.set(name, value, domain="openrent.co.uk")
+    return session
+
+
+# ---------------------------------------------------------------------------
+# HTTP fetch with retry
+# ---------------------------------------------------------------------------
+
+
+def _status_label(code: int) -> str:
+    if code >= 500:
+        return "5xx"
+    return str(code)
+
+
+def fetch_page(
+    client: Session, url: str, max_retries: int = 3,
+) -> str | None:
+    """GET HTML with retries on 429/5xx. Returns None on permanent failure.
+    WAF challenge (202 or 403 with challenge JS) raises WafChallengeError."""
+    for attempt in range(max_retries):
+        try:
+            resp = client.get(url, timeout=30)
+            openrent_requests_total.labels(status=_status_label(resp.status_code)).inc()
+
+            if resp.status_code == 200:
+                html = resp.text
+                # Detect WAF challenge page masquerading as 200
+                if "AwsWafIntegration" in html and "challenge.js" in html:
+                    raise WafChallengeError("Got AWS WAF challenge page — cookies expired")
+                return html
+
+            if resp.status_code in (202, 403):
+                raise WafChallengeError(f"HTTP {resp.status_code} — cookies likely expired")
+
+            if resp.status_code in (429, 500, 502, 503, 504):
+                delay = RETRY_BASE_DELAY * (2 ** attempt)
+                log.warning(
+                    "HTTP %d from %s, retry %d/%d in %.1fs",
+                    resp.status_code, url, attempt + 1, max_retries, delay,
+                )
+                time.sleep(delay)
+                continue
+
+            log.error("HTTP %d from %s (non-retryable)", resp.status_code, url)
+            return None
+
+        except WafChallengeError:
+            raise
+        except RequestsError as e:
+            openrent_errors_total.labels(type=type(e).__name__).inc()
+            delay = RETRY_BASE_DELAY * (2 ** attempt)
+            log.warning(
+                "%s from %s, retry %d/%d in %.1fs",
+                type(e).__name__, url, attempt + 1, max_retries, delay,
+            )
+            time.sleep(delay)
+
+    openrent_errors_total.labels(type="retry_exhausted").inc()
+    log.error("All %d retries exhausted for %s", max_retries, url)
+    return None
+
+
+# ---------------------------------------------------------------------------
+# HTML parsing
+# ---------------------------------------------------------------------------
+
+
+def _extract_price(text: str) -> tuple[int, str] | None:
+    """Extract price and frequency from text like '£1,500 pcm' or '£350 pw'.
+    Returns (price_int, frequency) or None."""
+    match = re.search(r"£([\d,]+)", text)
+    if not match:
+        return None
+    price = int(match.group(1).replace(",", ""))
+    lower = text.lower()
+    if "pw" in lower or "per week" in lower or "/w" in lower:
+        return price, "weekly"
+    if "pa" in lower or "per annum" in lower or "/y" in lower:
+        return price, "yearly"
+    # OpenRent defaults to pcm (per calendar month)
+    return price, "monthly"
+
+
+def _extract_bedrooms_from_title(title: str) -> int | None:
+    """Extract bedroom count from title like '2 Bed Flat, Pimlico'."""
+    match = re.search(r"(\d+)\s*bed", title, re.IGNORECASE)
+    if match:
+        return int(match.group(1))
+    if re.search(r"\bstudio\b", title, re.IGNORECASE):
+        return 0
+    return None
+
+
+def _extract_postcode(text: str) -> str | None:
+    """Extract full UK postcode from text like '2 Bed Flat, Pimlico, SW1V 2AA'."""
+    match = re.search(r"([A-Z]{1,2}\d[A-Z0-9]?\s*\d[A-Z]{2})", text, re.IGNORECASE)
+    if match:
+        return match.group(1).upper().strip()
+    return None
+
+
+def parse_search_results(html: str) -> list[dict]:
+    """Parse property data from OpenRent search results HTML.
+
+    Returns list of raw property dicts extracted from property cards.
+    Uses multiple fallback selectors for resilience against markup changes.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    properties = []
+
+    # Try known selectors for property cards (most specific first)
+    cards = soup.select("a.pli")
+    if not cards:
+        cards = soup.select(".pli.clearfix")
+    if not cards:
+        cards = soup.select("[class*='propertyListing']")
+    if not cards:
+        # Last resort: look for links that match property URL pattern
+        cards = soup.find_all("a", href=re.compile(r"/property-to-rent/"))
+
+    if not cards:
+        log.warning(
+            "No property cards found in search HTML (%d bytes). "
+            "CSS selectors may need updating.",
+            len(html),
+        )
+        return []
+
+    for card in cards:
+        prop: dict = {}
+
+        # Extract property URL and ID from href
+        href = card.get("href", "")
+        if not href:
+            continue
+
+        prop["url"] = href if href.startswith("http") else OPENRENT_BASE + href
+        id_match = re.search(r"/(\d+)(?:\?|$|#)", href)
+        if id_match:
+            prop["id"] = id_match.group(1)
+        else:
+            continue  # can't use a property without an ID
+
+        # Extract card text for parsing
+        card_text = card.get_text(" ", strip=True)
+
+        # Price
+        price_result = _extract_price(card_text)
+        if price_result:
+            prop["price"], prop["frequency"] = price_result
+
+        # Title / address — try specific elements first, fall back to card text
+        title_el = card.select_one(
+            ".listing-title, .banda, h2, h3, [class*='title']"
+        )
+        prop["title"] = (
+            title_el.get_text(strip=True) if title_el
+            else card_text[:200]
+        )
+
+        # Bedrooms from title text
+        beds = _extract_bedrooms_from_title(prop["title"])
+        if beds is not None:
+            prop["bedrooms"] = beds
+
+        # Postcode from title
+        postcode = _extract_postcode(prop["title"])
+        if postcode:
+            prop["postcode"] = postcode
+
+        # Coordinates from data attributes (if present on card or child elements)
+        for el in [card] + card.select("[data-lat], [data-latitude]"):
+            lat = el.get("data-lat") or el.get("data-latitude")
+            lng = (
+                el.get("data-lng")
+                or el.get("data-longitude")
+                or el.get("data-lon")
+            )
+            if lat and lng:
+                try:
+                    prop["lat"] = float(lat)
+                    prop["lng"] = float(lng)
+                except ValueError:
+                    pass
+                break
+
+        properties.append(prop)
+
+    log.debug("Parsed %d property cards from search HTML", len(properties))
+    return properties
+
+
+def parse_property_detail(html: str) -> dict:
+    """Parse a single property detail page for additional data.
+
+    Extracts: bedrooms, bathrooms, price, property_type, postcode,
+    lat/lng (from map data), description (for floor area).
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    details: dict = {}
+
+    # Parse structured data tables (class "table table-striped")
+    for table in soup.select("table.table-striped, table.table"):
+        for row in table.select("tr"):
+            cells = row.select("td, th")
+            if len(cells) < 2:
+                continue
+            label = cells[0].get_text(strip=True).lower()
+            value = cells[1].get_text(strip=True)
+
+            if "bedroom" in label:
+                match = re.search(r"(\d+)", value)
+                if match:
+                    details["bedrooms"] = int(match.group(1))
+            elif "bathroom" in label:
+                match = re.search(r"(\d+)", value)
+                if match:
+                    details["bathrooms"] = int(match.group(1))
+            elif "rent" in label or "price" in label:
+                match = re.search(r"£([\d,]+)", value)
+                if match:
+                    details["price"] = int(match.group(1).replace(",", ""))
+            elif "type" in label:
+                details["property_type"] = value
+            elif "available" in label or "move" in label:
+                details["available_date"] = value
+            elif "furnish" in label:
+                details["furnished"] = value
+
+    # Extract postcode from page title / address heading
+    title_tag = soup.select_one("h1, .property-title, [class*='title']")
+    if title_tag:
+        title_text = title_tag.get_text(strip=True)
+        details["title"] = title_text
+        postcode = _extract_postcode(title_text)
+        if postcode:
+            details["postcode"] = postcode
+
+    # Extract coordinates from map element data attributes
+    map_el = soup.select_one("[data-lat], [data-latitude]")
+    if map_el:
+        lat = map_el.get("data-lat") or map_el.get("data-latitude")
+        lng = (
+            map_el.get("data-lng")
+            or map_el.get("data-longitude")
+            or map_el.get("data-lon")
+        )
+        if lat and lng:
+            try:
+                details["lat"] = float(lat)
+                details["lng"] = float(lng)
+            except ValueError:
+                pass
+
+    # Also check for coordinates in JSON-LD or inline JavaScript
+    if "lat" not in details:
+        for script in soup.select("script"):
+            text = script.string or ""
+            lat_match = re.search(r'"latitude"\s*:\s*([\d.-]+)', text)
+            lng_match = re.search(r'"longitude"\s*:\s*([\d.-]+)', text)
+            if lat_match and lng_match:
+                try:
+                    details["lat"] = float(lat_match.group(1))
+                    details["lng"] = float(lng_match.group(1))
+                except ValueError:
+                    pass
+                break
+
+    # Extract description for floor area parsing
+    desc_el = soup.select_one(
+        ".description, [class*='description'], #description"
+    )
+    if desc_el:
+        details["description"] = desc_el.get_text(strip=True)
+
+    return details
+
+
+# ---------------------------------------------------------------------------
+# Property type mapping & floor area
+# ---------------------------------------------------------------------------
+
+
+def map_property_type(raw_type: str | None) -> str:
+    """Map OpenRent property type to canonical type."""
+    if not raw_type:
+        return "Other"
+    canonical = PROPERTY_TYPE_MAP.get(raw_type)
+    if canonical:
+        return canonical
+    lower = raw_type.lower()
+    if "room" in lower or "shared" in lower:
+        return "Other"
+    if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower:
+        return "Flats/Maisonettes"
+    if "detached" in lower and "semi" not in lower:
+        return "Detached"
+    if "semi" in lower:
+        return "Semi-Detached"
+    if "terrace" in lower or "mews" in lower:
+        return "Terraced"
+    if "house" in lower:
+        return "Detached"
+    log.debug("Unknown property type: %r — mapping to Other", raw_type)
+    return "Other"
+
+
+def parse_floor_area(description: str | None) -> float | None:
+    """Try to extract floor area from description text."""
+    if not description:
+        return None
+    m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*ft", description, re.IGNORECASE)
+    if m:
+        sqft = float(m.group(1).replace(",", ""))
+        return round(sqft * 0.092903, 1)
+    m = re.search(r"([\d,]+(?:\.\d+)?)\s*sq\.?\s*m", description, re.IGNORECASE)
+    if m:
+        return round(float(m.group(1).replace(",", "")), 1)
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Transform & search
+# ---------------------------------------------------------------------------
+
+
+def transform_property(
+    search_data: dict,
+    detail_data: dict | None,
+    pc_index: PostcodeSpatialIndex,
+    pc_coords: dict[str, tuple[float, float]],
+) -> dict | None:
+    """Transform OpenRent property data into our output schema.
+
+    Merges data from the search results page and (optionally) the detail page.
+    Uses pc_coords (postcode → lat/lng) as a fallback when coordinates are
+    missing but a postcode is available.
+    """
+    detail = detail_data or {}
+
+    # Merge: detail page data takes precedence
+    lat = detail.get("lat") or search_data.get("lat")
+    lng = detail.get("lng") or search_data.get("lng")
+    price = detail.get("price") or search_data.get("price")
+    if not price:
+        return None
+
+    frequency = search_data.get("frequency", "monthly")
+
+    # Get postcode: detail page > search card > spatial index
+    postcode = detail.get("postcode") or search_data.get("postcode")
+
+    if lat is not None and lng is not None:
+        # Validate coordinates are in England
+        if not (49 <= lat <= 56 and -7 <= lng <= 2):
+            log.debug("Coords outside England: lat=%.4f lng=%.4f — skipping", lat, lng)
+            return None
+        if not postcode:
+            postcode = pc_index.nearest(lat, lng)
+    elif postcode:
+        # Have postcode but no coordinates — look up centroid from arcgis data
+        coords = pc_coords.get(postcode)
+        if coords:
+            lat, lng = coords
+        else:
+            log.debug("Postcode %s not in arcgis data — skipping", postcode)
+            return None
+    else:
+        return None
+
+    if not postcode:
+        log.debug("No postcode for property — skipping")
+        return None
+
+    bedrooms = detail.get("bedrooms") or search_data.get("bedrooms", 0) or 0
+    bathrooms = detail.get("bathrooms", 0) or 0
+
+    title = detail.get("title") or search_data.get("title", "")
+    address = title.split(",")[0].strip() if title else ""
+
+    property_type = detail.get("property_type", "")
+    # Infer from title if not found in detail page
+    if not property_type and title:
+        lower = title.lower()
+        if "flat" in lower or "apartment" in lower:
+            property_type = "Flat"
+        elif "studio" in lower:
+            property_type = "Studio"
+        elif "maisonette" in lower:
+            property_type = "Maisonette"
+        elif "house" in lower:
+            property_type = "House"
+        elif "room" in lower:
+            property_type = "Room"
+
+    prop_id = search_data.get("id", "")
+    listing_url = search_data.get(
+        "url", f"{OPENRENT_BASE}/{prop_id}" if prop_id else "",
+    )
+    description = detail.get("description", "")
+
+    return {
+        "id": f"or_{prop_id}",
+        "Bedrooms": bedrooms,
+        "Bathrooms": bathrooms,
+        "Number of bedrooms & living rooms": bedrooms + bathrooms,
+        "lon": lng,
+        "lat": lat,
+        "Postcode": postcode,
+        "Address per Property Register": address,
+        "Leasehold/Freehold": None,
+        "Property type": map_property_type(property_type),
+        "Property sub-type": property_type or "Unknown",
+        "price": int(price),
+        "price_frequency": frequency,
+        "Price qualifier": "",
+        "Total floor area (sqm)": parse_floor_area(description),
+        "Listing URL": listing_url,
+        "Listing features": [],
+        "first_visible_date": detail.get("available_date", ""),
+    }
+
+
+def search_outcode(
+    client: Session,
+    outcode: str,
+    pc_index: PostcodeSpatialIndex,
+    pc_coords: dict[str, tuple[float, float]],
+    fetch_details: bool = True,
+) -> list[dict]:
+    """Search OpenRent for rental properties in one outcode.
+
+    1. Fetches the search results page for the outcode
+    2. Parses property cards from the HTML
+    3. Optionally fetches each property's detail page for full data
+    4. Transforms to common output schema
+
+    Args:
+        fetch_details: If True, visits each property's detail page for
+            coordinates and extra data. Slower but more complete.
+            If False, relies only on search card data + postcode lookup.
+    """
+    search_url = f"{OPENRENT_BASE}/properties-to-rent/?term={outcode}&isLive=true"
+
+    html = fetch_page(client, search_url)
+    if not html:
+        return []
+
+    search_results = parse_search_results(html)
+    if not search_results:
+        return []
+
+    properties = []
+    for search_data in search_results:
+        detail_data = None
+
+        if fetch_details and search_data.get("url"):
+            detail_html = fetch_page(client, search_data["url"])
+            if detail_html:
+                detail_data = parse_property_detail(detail_html)
+            # Shorter delay for detail pages (within same outcode)
+            time.sleep(DELAY_BETWEEN_PAGES * 0.5)
+
+        transformed = transform_property(
+            search_data, detail_data, pc_index, pc_coords,
+        )
+        if transformed:
+            properties.append(transformed)
+            openrent_properties_scraped.labels(channel="rent").inc()
+
+    return properties