"""Persistent detail-fetch caches that survive across scrape runs. Each portal recovers a listing's true postcode (Rightmove/OnTheMarket) or full geo dict (Zoopla) from its detail page. That value never changes for a given listing id, yet the in-memory caches are discarded at the end of every run — so each run re-fetches every listing's detail page from scratch. Persisting the cache to disk means a steady-state run only fetches NEWLY-appeared listings, typically a small fraction of the market, which is the single biggest saving for a recurring scrape (and it *reduces* request volume rather than adding to it). The stored values are always JSON-serialisable: Rightmove/OnTheMarket cache ``str | None`` postcodes; Zoopla caches a flat dict of primitives (or ``None``). A cached ``None`` (a listing whose detail page yielded nothing usable) is persisted too, so a postcode-less listing is not re-fetched every run. """ import json import logging import os import tempfile from pathlib import Path log = logging.getLogger("rightmove") def load_cache(path: str | Path) -> dict: """Load a persisted detail cache. Returns ``{}`` when absent or unreadable. A corrupt or non-object file is treated as empty rather than fatal — a bad cache must never block a scrape; the worst case is re-fetching details.""" p = Path(path) if not p.exists(): return {} try: with open(p, encoding="utf-8") as fh: data = json.load(fh) except (OSError, ValueError) as exc: log.warning("Could not read detail cache %s: %s", p, exc) return {} if not isinstance(data, dict): log.warning("Detail cache %s is not a JSON object; ignoring", p) return {} log.info("Loaded %d cached detail entries from %s", len(data), p) return data def save_cache(path: str | Path, data: dict) -> None: """Atomically write a detail cache to disk (temp file + ``os.replace``). Creates the parent directory if needed. Write failures are logged and swallowed: persisting the cache is an optimisation, never a hard requirement, so a read-only disk must not fail an otherwise-good scrape.""" p = Path(path) try: p.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=str(p.parent), suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as fh: json.dump(data, fh) os.replace(tmp, p) finally: if os.path.exists(tmp): os.unlink(tmp) log.info("Saved %d cached detail entries to %s", len(data), p) except OSError as exc: log.warning("Could not write detail cache %s: %s", p, exc)