import logging import random import threading import time from urllib.parse import urlsplit import httpx from fake_useragent import UserAgent import gluetun import shutdown from constants import ( BLOCK_403_THRESHOLD, BLOCK_MAX_ROTATIONS, GLUETUN_PROXY, MAX_RETRIES, REQUESTS_PER_SECOND, RETRY_BASE_DELAY, ) log = logging.getLogger("rightmove") class BlockedError(RuntimeError): """A host is refusing our egress IP and rotating away from it did not help. Raised rather than returned so a block can never be mistaken for "this query has no results": that conflation is precisely what turned Rightmove's 2026-07-15 edge block into a silent 0-listing run (see BLOCK_403_THRESHOLD). Callers should abandon the source, not the outcode. """ class RateLimiter: """Thread-safe global limiter: spaces request starts by a minimum interval. Detail-page fetches run concurrently across many worker threads (and across providers), but a single shared limiter caps their COMBINED rate so the VPN egress IP stays polite. Each ``acquire()`` reserves the next free time slot under a lock, then sleeps (outside the lock) until that slot, so N threads calling concurrently are spaced ``1/rate_per_second`` apart rather than all firing at once. ``rate_per_second <= 0`` disables limiting.""" def __init__(self, rate_per_second: float): self._interval = 1.0 / rate_per_second if rate_per_second > 0 else 0.0 self._lock = threading.Lock() self._next = 0.0 def acquire(self) -> None: if self._interval <= 0: return with self._lock: now = time.monotonic() if now >= self._next: self._next = now + self._interval wait = 0.0 else: wait = self._next - now self._next += self._interval if wait > 0: shutdown.sleep(wait) # Shared by every HTTP-based fetch (search pages and detail pages, all # providers). Spacing is global, so politeness is decoupled from concurrency. RATE_LIMITER = RateLimiter(REQUESTS_PER_SECOND) class _EgressBlockTracker: """Escalates a run of 403s from one host: rotate the egress IP, then give up. Counts CONSECUTIVE 403s per host, reset by any 200 from that host, so one forbidden URL never trips it while a refused egress IP does within a couple of calls. The rotation budget is per-run and shared across hosts and threads, because the tunnel being rotated is shared too. """ def __init__(self, threshold: int, max_rotations: int): self._threshold = max(threshold, 1) self._max_rotations = max(max_rotations, 0) self._lock = threading.Lock() self._consecutive: dict[str, int] = {} self._rotations = 0 def record_success(self, host: str) -> None: with self._lock: self._consecutive.pop(host, None) def record_403(self, host: str) -> str: """Register a 403 and decide what the caller should do next. Returns "retry" (below the threshold, back off normally), "rotated" (the egress IP changed, so the call deserves a fresh retry budget) or "blocked" (the rotation budget is spent, or rotating failed). Rotation runs under the lock: concurrent threads caught in the same 403 storm queue behind ONE tunnel restart and then re-read the reset counters, rather than each spending a slice of the budget. """ with self._lock: count = self._consecutive.get(host, 0) + 1 self._consecutive[host] = count if count < self._threshold: return "retry" if self._rotations >= self._max_rotations: return "blocked" self._rotations += 1 log.warning( "%d consecutive 403s from %s; rotating egress IP (rotation %d/%d)", count, host, self._rotations, self._max_rotations, ) rotated = gluetun.rotate_ip() # Either way the counters restart: on success the next 403s are # evidence about the NEW IP, and on failure we must not re-enter # rotation on the very next 403. self._consecutive.clear() if not rotated: log.error("Egress IP rotation failed; treating %s as blocked", host) return "blocked" return "rotated" def reset(self) -> None: """Forget all counters and refund the rotation budget (tests only).""" with self._lock: self._consecutive.clear() self._rotations = 0 BLOCK_TRACKER = _EgressBlockTracker(BLOCK_403_THRESHOLD, BLOCK_MAX_ROTATIONS) _ua = UserAgent( browsers=["Chrome", "Edge"], os=["Windows", "Mac OS X"], min_version=120.0 ) def make_client() -> httpx.Client: # Route through the Gluetun HTTP proxy (VPN egress) when configured. return httpx.Client( timeout=30, headers={"User-Agent": _ua.random, "Accept": "application/json"}, follow_redirects=True, proxy=GLUETUN_PROXY or None, ) def fetch_with_retry( client: httpx.Client, url: str, params: dict | None = None, on_403: bool = True ) -> dict | None: """GET JSON with retries on 403/429/5xx/connection errors. Returns None on permanent failure, EXCEPT when the host is refusing our egress IP, which raises BlockedError instead. A 403 from a shared CDN edge is usually transient and is retried with backoff; once one host has 403'd BLOCK_403_THRESHOLD times in a row the egress IP is rotated (and this call gets a fresh retry budget on the new IP), and once the rotation budget is spent the block is raised rather than degraded to a None the caller would read as "no results". ``on_403=False`` opts out for callers where a 403 is an expected per-URL answer rather than a verdict on our IP. """ host = urlsplit(url).netloc attempt = 0 while attempt < MAX_RETRIES: if shutdown.stop_requested(): return None try: RATE_LIMITER.acquire() resp = client.get(url, params=params) if resp.status_code == 200: BLOCK_TRACKER.record_success(host) return resp.json() if resp.status_code == 403 and on_403: action = BLOCK_TRACKER.record_403(host) if action == "blocked": raise BlockedError( f"HTTP 403 from {url}: {host} is refusing this egress IP " f"and rotating away from it did not help" ) delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1) log.warning( "HTTP 403 from %s, retry %d/%d in %.1fs", url, attempt + 1, MAX_RETRIES, delay, ) # A rotation means a different egress IP, so the failures this # call already accrued say nothing about the new one. attempt = 0 if action == "rotated" else attempt + 1 shutdown.sleep(delay) continue if resp.status_code in (429, 500, 502, 503, 504): delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1) log.warning( "HTTP %d from %s, retry %d/%d in %.1fs", resp.status_code, url, attempt + 1, MAX_RETRIES, delay, ) attempt += 1 shutdown.sleep(delay) continue log.error("HTTP %d from %s (non-retryable)", resp.status_code, url) return None except ( httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout, httpx.PoolTimeout, ) as e: delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1) log.warning( "%s from %s, retry %d/%d in %.1fs", type(e).__name__, url, attempt + 1, MAX_RETRIES, delay, ) attempt += 1 shutdown.sleep(delay) log.error("All %d retries exhausted for %s", MAX_RETRIES, url) return None