Cache postcodes
This commit is contained in:
parent
f59d01227b
commit
8e4c56bb0d
7 changed files with 502 additions and 141 deletions
|
|
@ -1,20 +1,23 @@
|
|||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import httpx
|
||||
|
||||
import shutdown
|
||||
from constants import (
|
||||
DETAIL_FETCH_CONCURRENCY,
|
||||
PAGE_SIZE,
|
||||
DELAY_BETWEEN_PAGES,
|
||||
RIGHTMOVE_ACCURATE_PIN_TYPE,
|
||||
RIGHTMOVE_DETAIL_URL,
|
||||
RIGHTMOVE_FETCH_DETAILS,
|
||||
RIGHTMOVE_MAX_DETAILS_PER_OUTCODE,
|
||||
RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS,
|
||||
SEARCH_URL,
|
||||
TYPEAHEAD_URL,
|
||||
)
|
||||
from http_client import fetch_with_retry
|
||||
from http_client import RATE_LIMITER, fetch_with_retry
|
||||
from spatial import PostcodeSpatialIndex
|
||||
from transform import extract_full_postcode, normalize_postcode, transform_property
|
||||
|
||||
|
|
@ -168,16 +171,32 @@ def parse_detail_postcode(html: str) -> str | None:
|
|||
|
||||
# listingId -> true full postcode (or None when unavailable). Failures are
|
||||
# cached too, so a broken/duplicate listing is fetched at most once per run (the
|
||||
# same listing can reappear across overlapping outcode searches).
|
||||
# same listing can reappear across overlapping outcode searches). Seeded from /
|
||||
# dumped to a persistent on-disk cache by the orchestrator (see
|
||||
# postcode_cache.py) so a recurring scrape only re-fetches newly-listed
|
||||
# properties rather than every listing every run.
|
||||
_detail_postcode_cache: dict[str, str | None] = {}
|
||||
|
||||
|
||||
def seed_detail_cache(mapping: dict) -> None:
|
||||
"""Pre-populate the in-memory detail cache from a persisted snapshot."""
|
||||
if mapping:
|
||||
_detail_postcode_cache.update(mapping)
|
||||
|
||||
|
||||
def detail_cache_snapshot() -> dict:
|
||||
"""Return a JSON-serialisable copy of the in-memory detail cache."""
|
||||
return dict(_detail_postcode_cache)
|
||||
|
||||
|
||||
def _fetch_detail_postcode(client: httpx.Client, property_id: str) -> str | None:
|
||||
"""GET a listing detail page and return its true full postcode (or None).
|
||||
|
||||
Results (including failures) are cached by listing id. The detail page is a
|
||||
plain HTML GET — no Cloudflare, unlike Zoopla — so a single httpx call
|
||||
suffices; any error degrades gracefully to the coordinate fallback."""
|
||||
suffices; any error degrades gracefully to the coordinate fallback. Safe to
|
||||
call concurrently: distinct listing ids write distinct cache keys, and the
|
||||
shared RATE_LIMITER spaces the GETs."""
|
||||
if not property_id:
|
||||
return None
|
||||
if property_id in _detail_postcode_cache:
|
||||
|
|
@ -186,6 +205,7 @@ def _fetch_detail_postcode(client: httpx.Client, property_id: str) -> str | None
|
|||
postcode: str | None = None
|
||||
url = RIGHTMOVE_DETAIL_URL.format(id=property_id)
|
||||
try:
|
||||
RATE_LIMITER.acquire()
|
||||
resp = client.get(url, headers={"Accept": "text/html"})
|
||||
if resp.status_code == 200:
|
||||
postcode = parse_detail_postcode(resp.text)
|
||||
|
|
@ -219,53 +239,83 @@ def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _detail_postcode_for(
|
||||
def _needs_detail_fetch(prop: dict) -> bool:
|
||||
"""Whether a search result still needs a detail-page fetch for its postcode.
|
||||
|
||||
Skips listings the search already pins precisely: an "ACCURATE_POINT"
|
||||
``pinType`` means rooftop-exact coordinates, so the coordinate-nearest
|
||||
postcode is trustworthy and the detail page would only confirm it. Listings
|
||||
with an approximate pin — or no ``pinType`` field at all — still get fetched,
|
||||
so this degrades safely to the previous behaviour when the search payload
|
||||
omits ``pinType``."""
|
||||
if not RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS:
|
||||
return True
|
||||
loc = prop.get("location") or {}
|
||||
return loc.get("pinType") != RIGHTMOVE_ACCURATE_PIN_TYPE
|
||||
|
||||
|
||||
def _prime_detail_postcodes(
|
||||
client: httpx.Client,
|
||||
prop: dict,
|
||||
props: list[dict],
|
||||
fetch_details: bool,
|
||||
detail_budget: dict,
|
||||
) -> str | None:
|
||||
"""Look up a listing's true postcode, honouring the per-outcode fetch cap.
|
||||
detail_cap: int,
|
||||
) -> None:
|
||||
"""Fill ``_detail_postcode_cache`` for the listings that need a detail page.
|
||||
|
||||
Cached listings are always served (they cost neither a cap slot nor a GET);
|
||||
a fresh fetch is made only while ``detail_budget['remaining'] > 0``."""
|
||||
if not fetch_details:
|
||||
return None
|
||||
property_id = str(prop.get("id") or "")
|
||||
if not property_id:
|
||||
return None
|
||||
if property_id in _detail_postcode_cache:
|
||||
return _detail_postcode_cache[property_id]
|
||||
if detail_budget["remaining"] <= 0:
|
||||
return None
|
||||
detail_budget["remaining"] -= 1
|
||||
postcode = _fetch_detail_postcode(client, property_id)
|
||||
time.sleep(DELAY_BETWEEN_PAGES)
|
||||
return postcode
|
||||
Picks the fresh (uncached, not-skipped) listings — up to ``detail_cap`` per
|
||||
outcode — then fetches their detail pages CONCURRENTLY, bounded by
|
||||
``DETAIL_FETCH_CONCURRENCY`` (the shared RATE_LIMITER keeps the combined
|
||||
request rate polite). Cached listings cost neither a slot nor a GET. The
|
||||
worklist is deduplicated, so distinct ids write distinct cache keys and the
|
||||
concurrent fetches never race on the same key."""
|
||||
if not fetch_details or detail_cap <= 0:
|
||||
return
|
||||
|
||||
worklist: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for prop in props:
|
||||
property_id = str(prop.get("id") or "")
|
||||
if not property_id or property_id in seen:
|
||||
continue
|
||||
seen.add(property_id)
|
||||
if property_id in _detail_postcode_cache:
|
||||
continue
|
||||
if not _needs_detail_fetch(prop):
|
||||
continue
|
||||
worklist.append(property_id)
|
||||
if len(worklist) >= detail_cap:
|
||||
break
|
||||
|
||||
if not worklist:
|
||||
return
|
||||
|
||||
workers = min(DETAIL_FETCH_CONCURRENCY, len(worklist))
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
# Drain the iterator so every fetch runs and populates the cache here.
|
||||
for _ in pool.map(lambda pid: _fetch_detail_postcode(client, pid), worklist):
|
||||
pass
|
||||
|
||||
|
||||
def _paginate(
|
||||
def _collect_search_props(
|
||||
client: httpx.Client,
|
||||
outcode_id: str,
|
||||
outcode: str,
|
||||
channel_cfg: dict,
|
||||
pc_index: PostcodeSpatialIndex,
|
||||
max_properties: int | None = None,
|
||||
fetch_details: bool = False,
|
||||
detail_cap: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Paginate through search results. Returns (properties, result_count).
|
||||
"""Paginate the search API for one outcode+channel, collecting raw results.
|
||||
|
||||
When ``fetch_details`` is set, up to ``detail_cap`` listings per outcode have
|
||||
their detail page fetched for the property's TRUE full postcode (see
|
||||
``parse_detail_postcode``); the rest fall back to coordinate-derived
|
||||
postcodes."""
|
||||
properties = []
|
||||
Returns ``(raw_props, result_count)``. Pagination stays serial — each page
|
||||
reveals the next — but is cheap relative to detail fetching, and the
|
||||
RATE_LIMITER spaces the page GETs. Collection stops at ``max_properties`` raw
|
||||
listings, the end of results, or Rightmove's ``_MAX_INDEX`` page cap."""
|
||||
raw_props: list[dict] = []
|
||||
index = 0
|
||||
result_count = 0
|
||||
detail_budget = {"remaining": detail_cap}
|
||||
|
||||
while True:
|
||||
if shutdown.stop_requested():
|
||||
break
|
||||
params = {
|
||||
"useLocationIdentifier": "true",
|
||||
"locationIdentifier": f"OUTCODE^{outcode_id}",
|
||||
|
|
@ -284,37 +334,17 @@ def _paginate(
|
|||
)
|
||||
break
|
||||
|
||||
raw_props = data.get("properties", [])
|
||||
if not raw_props:
|
||||
page_props = data.get("properties", [])
|
||||
if not page_props:
|
||||
break
|
||||
raw_props.extend(page_props)
|
||||
|
||||
for prop in raw_props:
|
||||
try:
|
||||
detail_postcode = _detail_postcode_for(
|
||||
client, prop, fetch_details, detail_budget
|
||||
)
|
||||
transformed = transform_property(
|
||||
prop, outcode, pc_index, detail_postcode=detail_postcode
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Rightmove %s/%s property %s failed to transform: %s",
|
||||
outcode,
|
||||
channel_cfg["channel"],
|
||||
prop.get("id", "?"),
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
if transformed:
|
||||
properties.append(transformed)
|
||||
if max_properties is not None and len(properties) >= max_properties:
|
||||
return properties, result_count
|
||||
|
||||
# Check if there are more pages
|
||||
result_count_str = data.get("resultCount", "0")
|
||||
result_count = int(result_count_str.replace(",", ""))
|
||||
index += PAGE_SIZE
|
||||
|
||||
if max_properties is not None and len(raw_props) >= max_properties:
|
||||
break
|
||||
index += PAGE_SIZE
|
||||
if index >= result_count:
|
||||
break
|
||||
if index >= _MAX_INDEX:
|
||||
|
|
@ -327,7 +357,55 @@ def _paginate(
|
|||
)
|
||||
break
|
||||
|
||||
time.sleep(DELAY_BETWEEN_PAGES)
|
||||
return raw_props, result_count
|
||||
|
||||
|
||||
def _paginate(
|
||||
client: httpx.Client,
|
||||
outcode_id: str,
|
||||
outcode: str,
|
||||
channel_cfg: dict,
|
||||
pc_index: PostcodeSpatialIndex,
|
||||
max_properties: int | None = None,
|
||||
fetch_details: bool = False,
|
||||
detail_cap: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Collect search results, recover true postcodes, and transform them.
|
||||
|
||||
Search pages are paginated serially; then — when ``fetch_details`` is set —
|
||||
up to ``detail_cap`` listings per outcode have their detail page fetched
|
||||
CONCURRENTLY for the property's TRUE full postcode (see
|
||||
``parse_detail_postcode``), with listings the search already pins precisely
|
||||
skipped (see ``_needs_detail_fetch``). The rest fall back to
|
||||
coordinate-derived postcodes. Returns ``(properties, result_count)``."""
|
||||
raw_props, result_count = _collect_search_props(
|
||||
client, outcode_id, outcode, channel_cfg, max_properties
|
||||
)
|
||||
_prime_detail_postcodes(client, raw_props, fetch_details, detail_cap)
|
||||
|
||||
properties: list[dict] = []
|
||||
for prop in raw_props:
|
||||
property_id = str(prop.get("id") or "")
|
||||
detail_postcode = (
|
||||
_detail_postcode_cache.get(property_id) if fetch_details else None
|
||||
)
|
||||
try:
|
||||
transformed = transform_property(
|
||||
prop, outcode, pc_index, detail_postcode=detail_postcode
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Rightmove %s/%s property %s failed to transform: %s",
|
||||
outcode,
|
||||
channel_cfg["channel"],
|
||||
prop.get("id", "?"),
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
if transformed:
|
||||
properties.append(transformed)
|
||||
if max_properties is not None and len(properties) >= max_properties:
|
||||
break
|
||||
|
||||
return properties, result_count
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue