229 lines
8.3 KiB
Python
229 lines
8.3 KiB
Python
import logging
|
|
import random
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
|
|
import httpx
|
|
from fake_useragent import UserAgent
|
|
|
|
import gluetun
|
|
import shutdown
|
|
from constants import (
|
|
BLOCK_403_THRESHOLD,
|
|
BLOCK_MAX_ROTATIONS,
|
|
GLUETUN_PROXY,
|
|
MAX_RETRIES,
|
|
REQUESTS_PER_SECOND,
|
|
RETRY_BASE_DELAY,
|
|
)
|
|
|
|
log = logging.getLogger("rightmove")
|
|
|
|
|
|
class BlockedError(RuntimeError):
|
|
"""A host is refusing our egress IP and rotating away from it did not help.
|
|
|
|
Raised rather than returned so a block can never be mistaken for "this query
|
|
has no results": that conflation is precisely what turned Rightmove's
|
|
2026-07-15 edge block into a silent 0-listing run (see BLOCK_403_THRESHOLD).
|
|
Callers should abandon the source, not the outcode.
|
|
"""
|
|
|
|
|
|
class RateLimiter:
|
|
"""Thread-safe global limiter: spaces request starts by a minimum interval.
|
|
|
|
Detail-page fetches run concurrently across many worker threads (and across
|
|
providers), but a single shared limiter caps their COMBINED rate so the VPN
|
|
egress IP stays polite. Each ``acquire()`` reserves the next free time slot
|
|
under a lock, then sleeps (outside the lock) until that slot, so N threads
|
|
calling concurrently are spaced ``1/rate_per_second`` apart rather than all
|
|
firing at once. ``rate_per_second <= 0`` disables limiting."""
|
|
|
|
def __init__(self, rate_per_second: float):
|
|
self._interval = 1.0 / rate_per_second if rate_per_second > 0 else 0.0
|
|
self._lock = threading.Lock()
|
|
self._next = 0.0
|
|
|
|
def acquire(self) -> None:
|
|
if self._interval <= 0:
|
|
return
|
|
with self._lock:
|
|
now = time.monotonic()
|
|
if now >= self._next:
|
|
self._next = now + self._interval
|
|
wait = 0.0
|
|
else:
|
|
wait = self._next - now
|
|
self._next += self._interval
|
|
if wait > 0:
|
|
shutdown.sleep(wait)
|
|
|
|
|
|
# Shared by every HTTP-based fetch (search pages and detail pages, all
|
|
# providers). Spacing is global, so politeness is decoupled from concurrency.
|
|
RATE_LIMITER = RateLimiter(REQUESTS_PER_SECOND)
|
|
|
|
|
|
class _EgressBlockTracker:
|
|
"""Escalates a run of 403s from one host: rotate the egress IP, then give up.
|
|
|
|
Counts CONSECUTIVE 403s per host, reset by any 200 from that host, so one
|
|
forbidden URL never trips it while a refused egress IP does within a couple
|
|
of calls. The rotation budget is per-run and shared across hosts and
|
|
threads, because the tunnel being rotated is shared too.
|
|
"""
|
|
|
|
def __init__(self, threshold: int, max_rotations: int):
|
|
self._threshold = max(threshold, 1)
|
|
self._max_rotations = max(max_rotations, 0)
|
|
self._lock = threading.Lock()
|
|
self._consecutive: dict[str, int] = {}
|
|
self._rotations = 0
|
|
|
|
def record_success(self, host: str) -> None:
|
|
with self._lock:
|
|
self._consecutive.pop(host, None)
|
|
|
|
def record_403(self, host: str) -> str:
|
|
"""Register a 403 and decide what the caller should do next.
|
|
|
|
Returns "retry" (below the threshold, back off normally), "rotated"
|
|
(the egress IP changed, so the call deserves a fresh retry budget) or
|
|
"blocked" (the rotation budget is spent, or rotating failed).
|
|
|
|
Rotation runs under the lock: concurrent threads caught in the same 403
|
|
storm queue behind ONE tunnel restart and then re-read the reset
|
|
counters, rather than each spending a slice of the budget.
|
|
"""
|
|
with self._lock:
|
|
count = self._consecutive.get(host, 0) + 1
|
|
self._consecutive[host] = count
|
|
if count < self._threshold:
|
|
return "retry"
|
|
if self._rotations >= self._max_rotations:
|
|
return "blocked"
|
|
|
|
self._rotations += 1
|
|
log.warning(
|
|
"%d consecutive 403s from %s; rotating egress IP (rotation %d/%d)",
|
|
count,
|
|
host,
|
|
self._rotations,
|
|
self._max_rotations,
|
|
)
|
|
rotated = gluetun.rotate_ip()
|
|
# Either way the counters restart: on success the next 403s are
|
|
# evidence about the NEW IP, and on failure we must not re-enter
|
|
# rotation on the very next 403.
|
|
self._consecutive.clear()
|
|
|
|
if not rotated:
|
|
log.error("Egress IP rotation failed; treating %s as blocked", host)
|
|
return "blocked"
|
|
return "rotated"
|
|
|
|
def reset(self) -> None:
|
|
"""Forget all counters and refund the rotation budget (tests only)."""
|
|
with self._lock:
|
|
self._consecutive.clear()
|
|
self._rotations = 0
|
|
|
|
|
|
BLOCK_TRACKER = _EgressBlockTracker(BLOCK_403_THRESHOLD, BLOCK_MAX_ROTATIONS)
|
|
|
|
_ua = UserAgent(
|
|
browsers=["Chrome", "Edge"], os=["Windows", "Mac OS X"], min_version=120.0
|
|
)
|
|
|
|
|
|
def make_client() -> httpx.Client:
|
|
# Route through the Gluetun HTTP proxy (VPN egress) when configured.
|
|
return httpx.Client(
|
|
timeout=30,
|
|
headers={"User-Agent": _ua.random, "Accept": "application/json"},
|
|
follow_redirects=True,
|
|
proxy=GLUETUN_PROXY or None,
|
|
)
|
|
|
|
|
|
def fetch_with_retry(
|
|
client: httpx.Client, url: str, params: dict | None = None, on_403: bool = True
|
|
) -> dict | None:
|
|
"""GET JSON with retries on 403/429/5xx/connection errors.
|
|
|
|
Returns None on permanent failure, EXCEPT when the host is refusing our
|
|
egress IP, which raises BlockedError instead. A 403 from a shared CDN edge
|
|
is usually transient and is retried with backoff; once one host has 403'd
|
|
BLOCK_403_THRESHOLD times in a row the egress IP is rotated (and this call
|
|
gets a fresh retry budget on the new IP), and once the rotation budget is
|
|
spent the block is raised rather than degraded to a None the caller would
|
|
read as "no results". ``on_403=False`` opts out for callers where a 403 is
|
|
an expected per-URL answer rather than a verdict on our IP.
|
|
"""
|
|
host = urlsplit(url).netloc
|
|
attempt = 0
|
|
while attempt < MAX_RETRIES:
|
|
if shutdown.stop_requested():
|
|
return None
|
|
try:
|
|
RATE_LIMITER.acquire()
|
|
resp = client.get(url, params=params)
|
|
if resp.status_code == 200:
|
|
BLOCK_TRACKER.record_success(host)
|
|
return resp.json()
|
|
if resp.status_code == 403 and on_403:
|
|
action = BLOCK_TRACKER.record_403(host)
|
|
if action == "blocked":
|
|
raise BlockedError(
|
|
f"HTTP 403 from {url}: {host} is refusing this egress IP "
|
|
f"and rotating away from it did not help"
|
|
)
|
|
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
|
|
log.warning(
|
|
"HTTP 403 from %s, retry %d/%d in %.1fs",
|
|
url,
|
|
attempt + 1,
|
|
MAX_RETRIES,
|
|
delay,
|
|
)
|
|
# A rotation means a different egress IP, so the failures this
|
|
# call already accrued say nothing about the new one.
|
|
attempt = 0 if action == "rotated" else attempt + 1
|
|
shutdown.sleep(delay)
|
|
continue
|
|
if resp.status_code in (429, 500, 502, 503, 504):
|
|
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
|
|
log.warning(
|
|
"HTTP %d from %s, retry %d/%d in %.1fs",
|
|
resp.status_code,
|
|
url,
|
|
attempt + 1,
|
|
MAX_RETRIES,
|
|
delay,
|
|
)
|
|
attempt += 1
|
|
shutdown.sleep(delay)
|
|
continue
|
|
log.error("HTTP %d from %s (non-retryable)", resp.status_code, url)
|
|
return None
|
|
except (
|
|
httpx.ConnectError,
|
|
httpx.ReadTimeout,
|
|
httpx.WriteTimeout,
|
|
httpx.PoolTimeout,
|
|
) as e:
|
|
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
|
|
log.warning(
|
|
"%s from %s, retry %d/%d in %.1fs",
|
|
type(e).__name__,
|
|
url,
|
|
attempt + 1,
|
|
MAX_RETRIES,
|
|
delay,
|
|
)
|
|
attempt += 1
|
|
shutdown.sleep(delay)
|
|
log.error("All %d retries exhausted for %s", MAX_RETRIES, url)
|
|
return None
|