67 lines
2.6 KiB
Python
67 lines
2.6 KiB
Python
"""Persistent detail-fetch caches that survive across scrape runs.
|
|
|
|
Each portal recovers a listing's true postcode (Rightmove/OnTheMarket) or full
|
|
geo dict (Zoopla) from its detail page. That value never changes for a given
|
|
listing id, yet the in-memory caches are discarded at the end of every run — so
|
|
each run re-fetches every listing's detail page from scratch. Persisting the
|
|
cache to disk means a steady-state run only fetches NEWLY-appeared listings,
|
|
typically a small fraction of the market, which is the single biggest saving
|
|
for a recurring scrape (and it *reduces* request volume rather than adding to
|
|
it).
|
|
|
|
The stored values are always JSON-serialisable: Rightmove/OnTheMarket cache
|
|
``str | None`` postcodes; Zoopla caches a flat dict of primitives (or ``None``).
|
|
A cached ``None`` (a listing whose detail page yielded nothing usable) is
|
|
persisted too, so a postcode-less listing is not re-fetched every run.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
log = logging.getLogger("rightmove")
|
|
|
|
|
|
def load_cache(path: str | Path) -> dict:
|
|
"""Load a persisted detail cache. Returns ``{}`` when absent or unreadable.
|
|
|
|
A corrupt or non-object file is treated as empty rather than fatal — a bad
|
|
cache must never block a scrape; the worst case is re-fetching details."""
|
|
p = Path(path)
|
|
if not p.exists():
|
|
return {}
|
|
try:
|
|
with open(p, encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
log.warning("Could not read detail cache %s: %s", p, exc)
|
|
return {}
|
|
if not isinstance(data, dict):
|
|
log.warning("Detail cache %s is not a JSON object; ignoring", p)
|
|
return {}
|
|
log.info("Loaded %d cached detail entries from %s", len(data), p)
|
|
return data
|
|
|
|
|
|
def save_cache(path: str | Path, data: dict) -> None:
|
|
"""Atomically write a detail cache to disk (temp file + ``os.replace``).
|
|
|
|
Creates the parent directory if needed. Write failures are logged and
|
|
swallowed: persisting the cache is an optimisation, never a hard
|
|
requirement, so a read-only disk must not fail an otherwise-good scrape."""
|
|
p = Path(path)
|
|
try:
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(dir=str(p.parent), suffix=".tmp")
|
|
try:
|
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
json.dump(data, fh)
|
|
os.replace(tmp, p)
|
|
finally:
|
|
if os.path.exists(tmp):
|
|
os.unlink(tmp)
|
|
log.info("Saved %d cached detail entries to %s", len(data), p)
|
|
except OSError as exc:
|
|
log.warning("Could not write detail cache %s: %s", p, exc)
|