andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Cache postcodes

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-14 14:50:38 +01:00
parent f59d01227b
commit 8e4c56bb0d
7 changed files with 502 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

150
finder/onthemarket.py
View file

@ -40,17 +40,19 @@ import json
import logging
import random
import re
import time
from concurrent.futures import ThreadPoolExecutor
import httpx
import shutdown
from constants import (
DELAY_BETWEEN_PAGES,
DETAIL_FETCH_CONCURRENCY,
MAX_BEDROOMS,
MAX_RETRIES,
ONTHEMARKET_BASE,
RETRY_BASE_DELAY,
)
from http_client import RATE_LIMITER
from spatial import PostcodeSpatialIndex
from transform import (
clean_listing_address,
@ -89,10 +91,24 @@ _HTML_HEADERS = {
# listingId -> recovered full postcode (or None). Failures are cached too so a
# broken or postcode-less detail page is not re-fetched within a run (the same
# listing can reappear across overlapping outcode searches).
# listing can reappear across overlapping outcode searches). Seeded from /
# dumped to a persistent on-disk cache by the orchestrator (see
# postcode_cache.py) so a recurring scrape only re-fetches newly-listed
# properties rather than every listing every run.
_detail_postcode_cache: dict[str, str | None] = {}
def seed_detail_cache(mapping: dict) -> None:
"""Pre-populate the in-memory detail cache from a persisted snapshot."""
if mapping:
_detail_postcode_cache.update(mapping)
def detail_cache_snapshot() -> dict:
"""Return a JSON-serialisable copy of the in-memory detail cache."""
return dict(_detail_postcode_cache)
def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict | None:
"""GET one search-results page and return the embedded __NEXT_DATA__ JSON.
@ -103,7 +119,10 @@ def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict
params = {"page": str(page_num)} if page_num > 1 else None
for attempt in range(MAX_RETRIES):
if shutdown.stop_requested():
return None
try:
RATE_LIMITER.acquire()
resp = client.get(
url,
params=params,
@ -121,7 +140,7 @@ def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict
"%s from %s, retry %d/%d in %.1fs",
type(exc).__name__, url, attempt + 1, MAX_RETRIES, delay,
)
time.sleep(delay)
shutdown.sleep(delay)
continue
if 300 <= resp.status_code < 400:
@ -151,7 +170,7 @@ def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict
"HTTP %d from %s, retry %d/%d in %.1fs",
resp.status_code, url, attempt + 1, MAX_RETRIES, delay,
)
time.sleep(delay)
shutdown.sleep(delay)
continue
log.error("HTTP %d from %s (non-retryable)", resp.status_code, url)
return None
@ -218,7 +237,8 @@ def _fetch_detail_postcode(
reappears across overlapping outcode searches is fetched at most once. Plain
HTTPS GET — OnTheMarket detail pages have no Cloudflare challenge. Network /
parse errors degrade gracefully to None so the caller falls back to the
coordinate-nearest postcode.
coordinate-nearest postcode. Safe to call concurrently: distinct listing ids
write distinct cache keys, and the shared RATE_LIMITER spaces the GETs.
"""
if listing_id in _detail_postcode_cache:
return _detail_postcode_cache[listing_id]
@ -231,7 +251,10 @@ def _fetch_detail_postcode(
result: str | None = None
if full_url:
for attempt in range(MAX_RETRIES):
if shutdown.stop_requested():
break
try:
RATE_LIMITER.acquire()
resp = client.get(
full_url, headers=_HTML_HEADERS, follow_redirects=True
)
@ -246,7 +269,7 @@ def _fetch_detail_postcode(
"%s from %s, retry %d/%d in %.1fs",
type(exc).__name__, full_url, attempt + 1, MAX_RETRIES, delay,
)
time.sleep(delay)
shutdown.sleep(delay)
continue
if resp.status_code == 200:
@ -258,7 +281,7 @@ def _fetch_detail_postcode(
"HTTP %d from %s, retry %d/%d in %.1fs",
resp.status_code, full_url, attempt + 1, MAX_RETRIES, delay,
)
time.sleep(delay)
shutdown.sleep(delay)
continue
log.debug(
"OnTheMarket detail %s returned HTTP %d (no postcode)",
@ -422,6 +445,47 @@ def transform_property(
}
def _prime_detail_postcodes(
client: httpx.Client,
raw_listings: list[dict],
detail_cap: int,
) -> None:
"""Fill ``_detail_postcode_cache`` for the listings that need a detail page.
Picks the fresh (uncached) listings — up to ``detail_cap`` per outcode — then
fetches their detail pages CONCURRENTLY, bounded by
``DETAIL_FETCH_CONCURRENCY`` (the shared RATE_LIMITER keeps the combined
request rate polite). Cached listings cost neither a slot nor a GET. The
worklist is deduplicated, so distinct ids write distinct cache keys and the
concurrent fetches never race on the same key."""
if not OTM_FETCH_DETAILS or detail_cap <= 0:
return
worklist: list[tuple[str, str]] = [] # (details_url, listing_id)
seen: set[str] = set()
for raw in raw_listings:
listing_id = str(raw.get("id") or "")
if not listing_id or listing_id in seen:
continue
seen.add(listing_id)
if listing_id in _detail_postcode_cache:
continue
worklist.append((raw.get("details-url") or "", listing_id))
if len(worklist) >= detail_cap:
break
if not worklist:
return
workers = min(DETAIL_FETCH_CONCURRENCY, len(worklist))
with ThreadPoolExecutor(max_workers=workers) as pool:
# Drain the iterator so every fetch runs and populates the cache here.
for _ in pool.map(
lambda item: _fetch_detail_postcode(client, item[0], item[1]), worklist
):
pass
def search_outcode(
client: httpx.Client,
outcode: str,
@ -430,17 +494,19 @@ def search_outcode(
) -> list[dict]:
"""Paginate through OnTheMarket sale results for one outcode.
When ``OTM_FETCH_DETAILS`` is enabled, up to
``OTM_MAX_DETAILS_PER_OUTCODE`` listings per outcode have their detail page
fetched for the property's own postcode (see ``_fetch_detail_postcode``);
the rest fall back to the coordinate-nearest postcode.
Search pages are paginated serially (the RATE_LIMITER spaces the GETs); then,
when ``OTM_FETCH_DETAILS`` is enabled, up to ``OTM_MAX_DETAILS_PER_OUTCODE``
listings per outcode have their detail page fetched CONCURRENTLY for the
property's own postcode (see ``_fetch_detail_postcode``); the rest fall back
to the coordinate-nearest postcode.
"""
properties: list[dict] = []
raw_listings: list[dict] = []
seen_ids: set[str] = set()
page_num = 1
details_fetched = 0
while True:
if shutdown.stop_requested():
break
data = _fetch_page_json(client, outcode, page_num)
if data is None:
break
@ -453,47 +519,45 @@ def search_outcode(
)
break
raw_listings = state.get("list") or []
if not raw_listings:
page_listings = state.get("list") or []
if not page_listings:
break
for raw in raw_listings:
for raw in page_listings:
listing_id = str(raw.get("id") or "")
if listing_id and listing_id in seen_ids:
continue
seen_ids.add(listing_id)
raw_listings.append(raw)
detail_postcode = None
if OTM_FETCH_DETAILS and listing_id:
# Cached lookups are free; only fresh GETs count toward the cap
# and incur the inter-request delay.
cached = listing_id in _detail_postcode_cache
if cached or details_fetched < OTM_MAX_DETAILS_PER_OUTCODE:
detail_postcode = _fetch_detail_postcode(
client, raw.get("details-url") or "", listing_id
)
if not cached:
details_fetched += 1
time.sleep(DELAY_BETWEEN_PAGES)
try:
transformed = transform_property(raw, pc_index, detail_postcode)
except Exception as exc:
log.warning(
"OnTheMarket %s property %s failed to transform: %s",
outcode, listing_id or "?", exc,
)
continue
if transformed:
properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties:
return properties
if max_properties is not None and len(raw_listings) >= max_properties:
break
pagination = state.get("paginationControls") or {}
if not pagination.get("next"):
break
page_num += 1
time.sleep(DELAY_BETWEEN_PAGES)
_prime_detail_postcodes(client, raw_listings, OTM_MAX_DETAILS_PER_OUTCODE)
properties: list[dict] = []
for raw in raw_listings:
listing_id = str(raw.get("id") or "")
detail_postcode = (
_detail_postcode_cache.get(listing_id) if OTM_FETCH_DETAILS else None
)
try:
transformed = transform_property(raw, pc_index, detail_postcode)
except Exception as exc:
log.warning(
"OnTheMarket %s property %s failed to transform: %s",
outcode, listing_id or "?", exc,
)
continue
if transformed:
properties.append(transformed)
if max_properties is not None and len(properties) >= max_properties:
break
return properties