This commit is contained in:
Andras Schmelczer 2026-07-03 18:47:28 +01:00
parent ab688243d7
commit 463bd4c647
54 changed files with 13239 additions and 625 deletions

View file

@ -1,4 +1,4 @@
"""Zoopla (zoopla.co.uk) scraper sale properties.
"""Zoopla (zoopla.co.uk) scraper: sale properties.
Zoopla is behind Cloudflare Turnstile (managed interactive challenge), which
blocks non-browser HTTP clients and even Playwright with stealth patches. Only
@ -522,7 +522,7 @@ def _gluetun_set_vpn_status(client: httpx.Client, status: str) -> bool:
return False
if resp.status_code == 401:
log.warning(
"Gluetun vpn/status %s: 401 Unauthorized — the API key must be "
"Gluetun vpn/status %s: 401 Unauthorized. The API key must be "
"authorised for 'PUT /v1/vpn/status' in Gluetun's auth config.toml",
status,
)
@ -593,7 +593,7 @@ def _rotate_and_retry_challenge(page, max_rotations: int) -> bool:
"""Rotate IP and reload until the challenge clears. Returns True on success."""
for attempt in range(1, max_rotations + 1):
log.warning(
"Cloudflare Turnstile challenge rotating Gluetun IP (attempt %d/%d)",
"Cloudflare Turnstile challenge, rotating Gluetun IP (attempt %d/%d)",
attempt, max_rotations,
)
if not _rotate_gluetun_ip():
@ -637,7 +637,7 @@ def _wait_for_turnstile(page, headless_mode: bool | str) -> None:
if not _is_turnstile_challenge(page):
return
# Try Gluetun IP rotation first works in any mode and is the only option
# Try Gluetun IP rotation first: works in any mode and is the only option
# in headless/unattended runs where no human can click the challenge.
max_rotations = _gluetun_max_rotations()
if max_rotations > 0 and _rotate_and_retry_challenge(page, max_rotations):
@ -654,7 +654,7 @@ def _wait_for_turnstile(page, headless_mode: bool | str) -> None:
timeout = _challenge_timeout_seconds()
log.warning(
"Gluetun rotation insufficient — falling back to interactive solve. "
"Gluetun rotation insufficient. Falling back to interactive solve. "
"Complete the Cloudflare challenge in the Zoopla browser window; "
"waiting up to %ds. Profile: %s",
timeout,
@ -727,7 +727,7 @@ def launch_browser():
page.goto(f"{ZOOPLA_BASE}/", wait_until="domcontentloaded", timeout=60000)
_wait_for_turnstile(page, headless_mode)
log.info("Zoopla browser ready title: %s", page.title())
log.info("Zoopla browser ready, title: %s", page.title())
time.sleep(2)
# Dismiss cookie consent
@ -1117,14 +1117,14 @@ def _extract_outcode(text: str) -> str | None:
# "outcode":...,"postcode":...,"uprn":...} feeding the map widgets.
# Nearby points of interest (stations, schools, EV chargers) and comparable
# listings carry their own "coordinates" too, but never inside the property's
# own "location" / "address":{"fullAddress" wrapper so the wrapper, not a
# own "location" / "address":{"fullAddress" wrapper, so the wrapper, not a
# loose coordinates object, is what we anchor on (see parse_detail_geo).
# listingId -> parsed detail dict (or None). Failures are cached too, so a
# broken listing is not re-fetched within a run (the same listing reappears
# across overlapping outcode searches). Seeded from / dumped to a persistent
# on-disk cache by the orchestrator (see postcode_cache.py) so a recurring
# scrape only re-fetches newly-listed properties the biggest saving for
# scrape only re-fetches newly-listed properties, the biggest saving for
# Zoopla, whose detail fetch drives a real browser tab.
_detail_cache: dict[str, dict | None] = {}
@ -1144,7 +1144,7 @@ _LISTING_ID_RE = re.compile(r"/details/(\d+)/?")
# The property's own location is carried by a `"location":{...}` wrapper and a
# twin `"address":{"fullAddress":...}` widget object. We anchor on those
# wrappers (and capture their full object body, which contains exactly one
# nested object `coordinates`) rather than scanning for loose coordinate
# nested object, `coordinates`) rather than scanning for loose coordinate
# objects: nearby points of interest (stations/schools/EV chargers) and
# comparable/"similar" listings also embed coordinates, but never inside the
# property's own `"location"` / `"address":{"fullAddress"` wrapper, so the
@ -1225,7 +1225,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
# twin, not the `location` wrapper we anchor coordinates on. Pull it from
# the twin that shares this property's uprn; when there is no uprn to
# disambiguate, fall back to the first twin (document order = primary
# listing), but never guess a twin when a uprn exists and none matches
# listing), but never guess a twin when a uprn exists and none matches:
# that would risk grabbing a comparable listing's address.
if result is None or result.get("full_address"):
return result
@ -1245,7 +1245,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
result["full_address"] = first
return result
# Strategy 1 the property's own `location` wrapper (authoritative). Take
# Strategy 1: the property's own `location` wrapper (authoritative). Take
# the first match (the primary listing precedes any comparables in the
# flight stream), but prefer one whose outcode matches the searched outcode.
first_location = None
@ -1269,7 +1269,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
if first_location is not None:
return attach_full_address(first_location)
# Strategy 2 the `address` map-widget twin (same coordinates, backup).
# Strategy 2: the `address` map-widget twin (same coordinates, backup).
for match in _DETAIL_ADDRESS_RE.finditer(buf):
full_address = match.group(1) or None
body = match.group(2)
@ -1284,7 +1284,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
def _detail_cache_key(listing_url: str) -> str:
"""Cache key for a listing detail page its numeric id when present."""
"""Cache key for a listing detail page: its numeric id when present."""
id_match = _LISTING_ID_RE.search(listing_url)
return id_match.group(1) if id_match else listing_url
@ -1388,7 +1388,7 @@ def transform_property(
location comes from the listing's detail page (see ``parse_detail_geo`` /
``_fetch_listing_detail``), passed in as ``detail``. When detail-page
coordinates are available we resolve the nearest postcode via the spatial
index mirroring rightmove/onthemarket and only fall back to the coarse
index (mirroring rightmove/onthemarket) and only fall back to the coarse
outcode centroid when no detail location could be obtained."""
price = parse_int_value(raw.get("price")) or 0
@ -1563,7 +1563,7 @@ def search_outcode(
if detail_budget_seconds is not None:
detail_deadline = time.monotonic() + detail_budget_seconds
# Always try extraction even if result count is 0 the count regex may
# Always try extraction even if result count is 0: the count regex may
# not match Zoopla's current text format, but listings may still be in DOM
raw_listings = _paginate(
page,
@ -1577,7 +1577,7 @@ def search_outcode(
if not raw_listings:
if total_results > 0:
log.debug(
"Zoopla %s %s: page claims %d results but extraction found 0 "
"Zoopla %s %s: page claims %d results but extraction found 0; "
"DOM selectors may need updating",
outcode, "BUY", total_results,
)