441 lines
16 KiB
Python
441 lines
16 KiB
Python
import json
|
||
import logging
|
||
import re
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
import httpx
|
||
|
||
import shutdown
|
||
from constants import (
|
||
DETAIL_FETCH_CONCURRENCY,
|
||
PAGE_SIZE,
|
||
RIGHTMOVE_ACCURATE_PIN_TYPE,
|
||
RIGHTMOVE_DETAIL_URL,
|
||
RIGHTMOVE_FETCH_DETAILS,
|
||
RIGHTMOVE_MAX_DETAILS_PER_OUTCODE,
|
||
RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS,
|
||
SEARCH_URL,
|
||
TYPEAHEAD_URL,
|
||
)
|
||
from http_client import RATE_LIMITER, fetch_with_retry
|
||
from spatial import PostcodeSpatialIndex
|
||
from transform import extract_full_postcode, normalize_postcode, transform_property
|
||
|
||
log = logging.getLogger("rightmove")
|
||
|
||
# Outcode ID cache (Rightmove typeahead → internal ID)
|
||
outcode_cache: dict[str, str] = {}
|
||
|
||
# Rightmove hard-caps pagination at index 1008 (42 pages × 24 results).
|
||
# Requesting index >= 1008 returns HTTP 400.
|
||
_MAX_INDEX = 1008
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Detail-page postcode extraction
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# The search API (_paginate) only returns an outcode-level `displayAddress`
|
||
# (e.g. "Akerman Road, Brixton, London, SW9") — never the full postcode. Each
|
||
# listing's detail page, however, embeds the property's OWN full postcode in a
|
||
# `window.__PAGE_MODEL` script as `propertyData.address.{outcode, incode}`
|
||
# (e.g. outcode "SW9" + incode "0HD" → "SW9 0HD"), independently corroborated by
|
||
# `propertyData.propertyUrls.similarPropertiesUrl` ("/property-for-sale/SW9-0HD.html").
|
||
# This is the property's own postcode, NOT a nearest station/school: the
|
||
# `nearestStations`/`nearestAirports` arrays carry only names + distances, no
|
||
# postcodes, and the address outcode always matches the searched outcode.
|
||
# Recon over 24 live listings across SW9/E1/M1/LS6/E20 (incl. APPROXIMATE_POINT
|
||
# new-builds) found the full postcode present 100% of the time. There is no
|
||
# UPRN or house-number field anywhere in propertyData, so those stay None.
|
||
#
|
||
# __PAGE_MODEL is a "devalue"-style flattened object graph: its `data` field is
|
||
# a JSON STRING holding a flat array where every integer inside a container is
|
||
# an index reference into that same array (so the graph can dedupe). We
|
||
# brace-match the (large, deeply-nested) object literal — a non-greedy regex
|
||
# cannot — then rehydrate the reference graph before reading the address.
|
||
|
||
_PAGE_MODEL_RE = re.compile(r"window\.__PAGE_MODEL\s*=\s*")
|
||
|
||
|
||
def _extract_page_model_literal(html: str) -> str | None:
|
||
"""Return the `{...}` object literal assigned to window.__PAGE_MODEL.
|
||
|
||
Brace-matches with string/escape awareness so embedded braces and quotes in
|
||
string values don't end the match early. Returns None when absent."""
|
||
marker = _PAGE_MODEL_RE.search(html)
|
||
if not marker:
|
||
return None
|
||
start = marker.end()
|
||
if start >= len(html) or html[start] != "{":
|
||
return None
|
||
depth = 0
|
||
in_str = False
|
||
esc = False
|
||
for j in range(start, len(html)):
|
||
ch = html[j]
|
||
if in_str:
|
||
if esc:
|
||
esc = False
|
||
elif ch == "\\":
|
||
esc = True
|
||
elif ch == '"':
|
||
in_str = False
|
||
elif ch == '"':
|
||
in_str = True
|
||
elif ch == "{":
|
||
depth += 1
|
||
elif ch == "}":
|
||
depth -= 1
|
||
if depth == 0:
|
||
return html[start : j + 1]
|
||
return None
|
||
|
||
|
||
def _rehydrate(flat: list) -> object:
|
||
"""Resolve a devalue-style flattened reference array into a nested object.
|
||
|
||
Index 0 is the root; every int inside a dict/list is an index back into
|
||
``flat``. Memoised so shared/cyclic references resolve once."""
|
||
cache: dict[int, object] = {}
|
||
|
||
def resolve(idx: int) -> object:
|
||
if not isinstance(idx, int) or idx < 0 or idx >= len(flat):
|
||
return None
|
||
if idx in cache:
|
||
return cache[idx]
|
||
node = flat[idx]
|
||
if isinstance(node, dict):
|
||
out: dict = {}
|
||
cache[idx] = out
|
||
for key, value in node.items():
|
||
out[key] = resolve(value) if isinstance(value, int) else value
|
||
return out
|
||
if isinstance(node, list):
|
||
arr: list = []
|
||
cache[idx] = arr
|
||
for value in node:
|
||
arr.append(resolve(value) if isinstance(value, int) else value)
|
||
return arr
|
||
cache[idx] = node
|
||
return node
|
||
|
||
return resolve(0)
|
||
|
||
|
||
def parse_detail_postcode(html: str) -> str | None:
|
||
"""Extract a Rightmove property's TRUE full postcode from its detail HTML.
|
||
|
||
Pure and network-free so it is unit-testable: callers pass the page HTML.
|
||
Reads ``propertyData.address.outcode`` + ``.incode`` from window.__PAGE_MODEL
|
||
and returns a normalised full postcode (e.g. "SW9 0HD"), or None when the
|
||
page has no parseable address (the property location wrapper can be empty —
|
||
the caller then keeps the coordinate fallback). The returned outcode is
|
||
re-validated against the joined postcode so a malformed incode is dropped.
|
||
"""
|
||
if not html:
|
||
return None
|
||
literal = _extract_page_model_literal(html)
|
||
if not literal:
|
||
return None
|
||
try:
|
||
outer = json.loads(literal)
|
||
flat = json.loads(outer["data"])
|
||
except (ValueError, KeyError, TypeError):
|
||
return None
|
||
if not isinstance(flat, list) or not flat:
|
||
return None
|
||
|
||
root = _rehydrate(flat)
|
||
if not isinstance(root, dict):
|
||
return None
|
||
property_data = root.get("propertyData")
|
||
if not isinstance(property_data, dict):
|
||
return None
|
||
address = property_data.get("address")
|
||
if not isinstance(address, dict):
|
||
return None
|
||
|
||
outcode = address.get("outcode")
|
||
incode = address.get("incode")
|
||
if not isinstance(outcode, str) or not isinstance(incode, str):
|
||
return None
|
||
outcode, incode = outcode.strip(), incode.strip()
|
||
if not outcode or not incode:
|
||
return None
|
||
|
||
# Round-trip through the shared postcode validator/normaliser: this both
|
||
# canonicalises spacing and rejects an outcode/incode pair that doesn't form
|
||
# a structurally-valid UK postcode.
|
||
return extract_full_postcode(normalize_postcode(f"{outcode} {incode}"))
|
||
|
||
|
||
# listingId -> true full postcode (or None when unavailable). Failures are
|
||
# cached too, so a broken/duplicate listing is fetched at most once per run (the
|
||
# same listing can reappear across overlapping outcode searches). Seeded from /
|
||
# dumped to a persistent on-disk cache by the orchestrator (see
|
||
# postcode_cache.py) so a recurring scrape only re-fetches newly-listed
|
||
# properties rather than every listing every run.
|
||
_detail_postcode_cache: dict[str, str | None] = {}
|
||
|
||
|
||
def seed_detail_cache(mapping: dict) -> None:
|
||
"""Pre-populate the in-memory detail cache from a persisted snapshot."""
|
||
if mapping:
|
||
_detail_postcode_cache.update(mapping)
|
||
|
||
|
||
def detail_cache_snapshot() -> dict:
|
||
"""Return a JSON-serialisable copy of the in-memory detail cache."""
|
||
return dict(_detail_postcode_cache)
|
||
|
||
|
||
def _fetch_detail_postcode(client: httpx.Client, property_id: str) -> str | None:
|
||
"""GET a listing detail page and return its true full postcode (or None).
|
||
|
||
Results (including failures) are cached by listing id. The detail page is a
|
||
plain HTML GET — no Cloudflare, unlike Zoopla — so a single httpx call
|
||
suffices; any error degrades gracefully to the coordinate fallback. Safe to
|
||
call concurrently: distinct listing ids write distinct cache keys, and the
|
||
shared RATE_LIMITER spaces the GETs."""
|
||
if not property_id:
|
||
return None
|
||
if property_id in _detail_postcode_cache:
|
||
return _detail_postcode_cache[property_id]
|
||
|
||
postcode: str | None = None
|
||
url = RIGHTMOVE_DETAIL_URL.format(id=property_id)
|
||
try:
|
||
RATE_LIMITER.acquire()
|
||
resp = client.get(url, headers={"Accept": "text/html"})
|
||
if resp.status_code == 200:
|
||
postcode = parse_detail_postcode(resp.text)
|
||
else:
|
||
log.debug("Rightmove detail %s returned HTTP %d", url, resp.status_code)
|
||
except httpx.HTTPError as exc:
|
||
log.debug("Rightmove detail fetch failed %s: %s", url, exc)
|
||
|
||
_detail_postcode_cache[property_id] = postcode
|
||
return postcode
|
||
|
||
|
||
def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
|
||
"""Look up Rightmove's internal ID for an outcode via typeahead API."""
|
||
if outcode in outcode_cache:
|
||
return outcode_cache[outcode]
|
||
|
||
data = fetch_with_retry(
|
||
client, TYPEAHEAD_URL, {"query": outcode, "limit": "10", "exclude": "STREET"}
|
||
)
|
||
if not data:
|
||
return None
|
||
|
||
for match in data.get("matches", []):
|
||
if match.get("type") == "OUTCODE" and match.get("displayName") == outcode:
|
||
rid = str(match["id"])
|
||
outcode_cache[outcode] = rid
|
||
return rid
|
||
|
||
log.debug("Outcode %s not found in typeahead results", outcode)
|
||
return None
|
||
|
||
|
||
def _needs_detail_fetch(prop: dict) -> bool:
|
||
"""Whether a search result still needs a detail-page fetch for its postcode.
|
||
|
||
Skips listings the search already pins precisely: an "ACCURATE_POINT"
|
||
``pinType`` means rooftop-exact coordinates, so the coordinate-nearest
|
||
postcode is trustworthy and the detail page would only confirm it. Listings
|
||
with an approximate pin — or no ``pinType`` field at all — still get fetched,
|
||
so this degrades safely to the previous behaviour when the search payload
|
||
omits ``pinType``."""
|
||
if not RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS:
|
||
return True
|
||
loc = prop.get("location") or {}
|
||
return loc.get("pinType") != RIGHTMOVE_ACCURATE_PIN_TYPE
|
||
|
||
|
||
def _prime_detail_postcodes(
|
||
client: httpx.Client,
|
||
props: list[dict],
|
||
fetch_details: bool,
|
||
detail_cap: int,
|
||
) -> None:
|
||
"""Fill ``_detail_postcode_cache`` for the listings that need a detail page.
|
||
|
||
Picks the fresh (uncached, not-skipped) listings — up to ``detail_cap`` per
|
||
outcode — then fetches their detail pages CONCURRENTLY, bounded by
|
||
``DETAIL_FETCH_CONCURRENCY`` (the shared RATE_LIMITER keeps the combined
|
||
request rate polite). Cached listings cost neither a slot nor a GET. The
|
||
worklist is deduplicated, so distinct ids write distinct cache keys and the
|
||
concurrent fetches never race on the same key."""
|
||
if not fetch_details or detail_cap <= 0:
|
||
return
|
||
|
||
worklist: list[str] = []
|
||
seen: set[str] = set()
|
||
for prop in props:
|
||
property_id = str(prop.get("id") or "")
|
||
if not property_id or property_id in seen:
|
||
continue
|
||
seen.add(property_id)
|
||
if property_id in _detail_postcode_cache:
|
||
continue
|
||
if not _needs_detail_fetch(prop):
|
||
continue
|
||
worklist.append(property_id)
|
||
if len(worklist) >= detail_cap:
|
||
break
|
||
|
||
if not worklist:
|
||
return
|
||
|
||
workers = min(DETAIL_FETCH_CONCURRENCY, len(worklist))
|
||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||
# Drain the iterator so every fetch runs and populates the cache here.
|
||
for _ in pool.map(lambda pid: _fetch_detail_postcode(client, pid), worklist):
|
||
pass
|
||
|
||
|
||
def _collect_search_props(
|
||
client: httpx.Client,
|
||
outcode_id: str,
|
||
outcode: str,
|
||
channel_cfg: dict,
|
||
max_properties: int | None = None,
|
||
) -> tuple[list[dict], int]:
|
||
"""Paginate the search API for one outcode+channel, collecting raw results.
|
||
|
||
Returns ``(raw_props, result_count)``. Pagination stays serial — each page
|
||
reveals the next — but is cheap relative to detail fetching, and the
|
||
RATE_LIMITER spaces the page GETs. Collection stops at ``max_properties`` raw
|
||
listings, the end of results, or Rightmove's ``_MAX_INDEX`` page cap."""
|
||
raw_props: list[dict] = []
|
||
index = 0
|
||
result_count = 0
|
||
|
||
while True:
|
||
if shutdown.stop_requested():
|
||
break
|
||
params = {
|
||
"useLocationIdentifier": "true",
|
||
"locationIdentifier": f"OUTCODE^{outcode_id}",
|
||
"index": str(index),
|
||
"sortType": channel_cfg["sortType"],
|
||
"channel": channel_cfg["channel"],
|
||
"transactionType": channel_cfg["transactionType"],
|
||
}
|
||
data = fetch_with_retry(client, SEARCH_URL, params)
|
||
if not data:
|
||
log.warning(
|
||
"Failed to fetch index %d for %s/%s",
|
||
index,
|
||
outcode,
|
||
channel_cfg["channel"],
|
||
)
|
||
break
|
||
|
||
page_props = data.get("properties", [])
|
||
if not page_props:
|
||
break
|
||
raw_props.extend(page_props)
|
||
|
||
result_count_str = data.get("resultCount", "0")
|
||
result_count = int(result_count_str.replace(",", ""))
|
||
|
||
if max_properties is not None and len(raw_props) >= max_properties:
|
||
break
|
||
index += PAGE_SIZE
|
||
if index >= result_count:
|
||
break
|
||
if index >= _MAX_INDEX:
|
||
log.warning(
|
||
"%s/%s: %d results exceed Rightmove's %d-result page cap",
|
||
outcode,
|
||
channel_cfg["channel"],
|
||
result_count,
|
||
_MAX_INDEX,
|
||
)
|
||
break
|
||
|
||
return raw_props, result_count
|
||
|
||
|
||
def _paginate(
|
||
client: httpx.Client,
|
||
outcode_id: str,
|
||
outcode: str,
|
||
channel_cfg: dict,
|
||
pc_index: PostcodeSpatialIndex,
|
||
max_properties: int | None = None,
|
||
fetch_details: bool = False,
|
||
detail_cap: int = 0,
|
||
) -> tuple[list[dict], int]:
|
||
"""Collect search results, recover true postcodes, and transform them.
|
||
|
||
Search pages are paginated serially; then — when ``fetch_details`` is set —
|
||
up to ``detail_cap`` listings per outcode have their detail page fetched
|
||
CONCURRENTLY for the property's TRUE full postcode (see
|
||
``parse_detail_postcode``), with listings the search already pins precisely
|
||
skipped (see ``_needs_detail_fetch``). The rest fall back to
|
||
coordinate-derived postcodes. Returns ``(properties, result_count)``."""
|
||
raw_props, result_count = _collect_search_props(
|
||
client, outcode_id, outcode, channel_cfg, max_properties
|
||
)
|
||
_prime_detail_postcodes(client, raw_props, fetch_details, detail_cap)
|
||
|
||
properties: list[dict] = []
|
||
for prop in raw_props:
|
||
property_id = str(prop.get("id") or "")
|
||
detail_postcode = (
|
||
_detail_postcode_cache.get(property_id) if fetch_details else None
|
||
)
|
||
try:
|
||
transformed = transform_property(
|
||
prop, outcode, pc_index, detail_postcode=detail_postcode
|
||
)
|
||
except Exception as exc:
|
||
log.warning(
|
||
"Rightmove %s/%s property %s failed to transform: %s",
|
||
outcode,
|
||
channel_cfg["channel"],
|
||
prop.get("id", "?"),
|
||
exc,
|
||
)
|
||
continue
|
||
if transformed:
|
||
properties.append(transformed)
|
||
if max_properties is not None and len(properties) >= max_properties:
|
||
break
|
||
|
||
return properties, result_count
|
||
|
||
|
||
def search_outcode(
|
||
client: httpx.Client,
|
||
outcode_id: str,
|
||
outcode: str,
|
||
channel_cfg: dict,
|
||
pc_index: PostcodeSpatialIndex,
|
||
max_properties: int | None = None,
|
||
) -> list[dict]:
|
||
"""Paginate through unfiltered sale results for one outcode+channel.
|
||
|
||
Each listing's detail page is fetched for the property's TRUE full postcode
|
||
(gated by ``RIGHTMOVE_FETCH_DETAILS`` and capped per outcode by
|
||
``RIGHTMOVE_MAX_DETAILS_PER_OUTCODE``); listings beyond the cap keep the
|
||
coordinate-derived postcode."""
|
||
properties, _ = _paginate(
|
||
client,
|
||
outcode_id,
|
||
outcode,
|
||
channel_cfg,
|
||
pc_index,
|
||
max_properties=max_properties,
|
||
fetch_details=RIGHTMOVE_FETCH_DETAILS,
|
||
detail_cap=RIGHTMOVE_MAX_DETAILS_PER_OUTCODE,
|
||
)
|
||
|
||
if max_properties is not None and len(properties) >= max_properties:
|
||
return properties[:max_properties]
|
||
|
||
return properties
|