..
This commit is contained in:
parent
ab688243d7
commit
463bd4c647
54 changed files with 13239 additions and 625 deletions
|
|
@ -1,4 +1,4 @@
|
|||
"""Zoopla (zoopla.co.uk) scraper — sale properties.
|
||||
"""Zoopla (zoopla.co.uk) scraper: sale properties.
|
||||
|
||||
Zoopla is behind Cloudflare Turnstile (managed interactive challenge), which
|
||||
blocks non-browser HTTP clients and even Playwright with stealth patches. Only
|
||||
|
|
@ -522,7 +522,7 @@ def _gluetun_set_vpn_status(client: httpx.Client, status: str) -> bool:
|
|||
return False
|
||||
if resp.status_code == 401:
|
||||
log.warning(
|
||||
"Gluetun vpn/status %s: 401 Unauthorized — the API key must be "
|
||||
"Gluetun vpn/status %s: 401 Unauthorized. The API key must be "
|
||||
"authorised for 'PUT /v1/vpn/status' in Gluetun's auth config.toml",
|
||||
status,
|
||||
)
|
||||
|
|
@ -593,7 +593,7 @@ def _rotate_and_retry_challenge(page, max_rotations: int) -> bool:
|
|||
"""Rotate IP and reload until the challenge clears. Returns True on success."""
|
||||
for attempt in range(1, max_rotations + 1):
|
||||
log.warning(
|
||||
"Cloudflare Turnstile challenge — rotating Gluetun IP (attempt %d/%d)",
|
||||
"Cloudflare Turnstile challenge, rotating Gluetun IP (attempt %d/%d)",
|
||||
attempt, max_rotations,
|
||||
)
|
||||
if not _rotate_gluetun_ip():
|
||||
|
|
@ -637,7 +637,7 @@ def _wait_for_turnstile(page, headless_mode: bool | str) -> None:
|
|||
if not _is_turnstile_challenge(page):
|
||||
return
|
||||
|
||||
# Try Gluetun IP rotation first — works in any mode and is the only option
|
||||
# Try Gluetun IP rotation first: works in any mode and is the only option
|
||||
# in headless/unattended runs where no human can click the challenge.
|
||||
max_rotations = _gluetun_max_rotations()
|
||||
if max_rotations > 0 and _rotate_and_retry_challenge(page, max_rotations):
|
||||
|
|
@ -654,7 +654,7 @@ def _wait_for_turnstile(page, headless_mode: bool | str) -> None:
|
|||
|
||||
timeout = _challenge_timeout_seconds()
|
||||
log.warning(
|
||||
"Gluetun rotation insufficient — falling back to interactive solve. "
|
||||
"Gluetun rotation insufficient. Falling back to interactive solve. "
|
||||
"Complete the Cloudflare challenge in the Zoopla browser window; "
|
||||
"waiting up to %ds. Profile: %s",
|
||||
timeout,
|
||||
|
|
@ -727,7 +727,7 @@ def launch_browser():
|
|||
page.goto(f"{ZOOPLA_BASE}/", wait_until="domcontentloaded", timeout=60000)
|
||||
_wait_for_turnstile(page, headless_mode)
|
||||
|
||||
log.info("Zoopla browser ready — title: %s", page.title())
|
||||
log.info("Zoopla browser ready, title: %s", page.title())
|
||||
time.sleep(2)
|
||||
|
||||
# Dismiss cookie consent
|
||||
|
|
@ -1117,14 +1117,14 @@ def _extract_outcode(text: str) -> str | None:
|
|||
# "outcode":...,"postcode":...,"uprn":...} feeding the map widgets.
|
||||
# Nearby points of interest (stations, schools, EV chargers) and comparable
|
||||
# listings carry their own "coordinates" too, but never inside the property's
|
||||
# own "location" / "address":{"fullAddress" wrapper — so the wrapper, not a
|
||||
# own "location" / "address":{"fullAddress" wrapper, so the wrapper, not a
|
||||
# loose coordinates object, is what we anchor on (see parse_detail_geo).
|
||||
|
||||
# listingId -> parsed detail dict (or None). Failures are cached too, so a
|
||||
# broken listing is not re-fetched within a run (the same listing reappears
|
||||
# across overlapping outcode searches). Seeded from / dumped to a persistent
|
||||
# on-disk cache by the orchestrator (see postcode_cache.py) so a recurring
|
||||
# scrape only re-fetches newly-listed properties — the biggest saving for
|
||||
# scrape only re-fetches newly-listed properties, the biggest saving for
|
||||
# Zoopla, whose detail fetch drives a real browser tab.
|
||||
_detail_cache: dict[str, dict | None] = {}
|
||||
|
||||
|
|
@ -1144,7 +1144,7 @@ _LISTING_ID_RE = re.compile(r"/details/(\d+)/?")
|
|||
# The property's own location is carried by a `"location":{...}` wrapper and a
|
||||
# twin `"address":{"fullAddress":...}` widget object. We anchor on those
|
||||
# wrappers (and capture their full object body, which contains exactly one
|
||||
# nested object — `coordinates`) rather than scanning for loose coordinate
|
||||
# nested object, `coordinates`) rather than scanning for loose coordinate
|
||||
# objects: nearby points of interest (stations/schools/EV chargers) and
|
||||
# comparable/"similar" listings also embed coordinates, but never inside the
|
||||
# property's own `"location"` / `"address":{"fullAddress"` wrapper, so the
|
||||
|
|
@ -1225,7 +1225,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
|
|||
# twin, not the `location` wrapper we anchor coordinates on. Pull it from
|
||||
# the twin that shares this property's uprn; when there is no uprn to
|
||||
# disambiguate, fall back to the first twin (document order = primary
|
||||
# listing), but never guess a twin when a uprn exists and none matches —
|
||||
# listing), but never guess a twin when a uprn exists and none matches:
|
||||
# that would risk grabbing a comparable listing's address.
|
||||
if result is None or result.get("full_address"):
|
||||
return result
|
||||
|
|
@ -1245,7 +1245,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
|
|||
result["full_address"] = first
|
||||
return result
|
||||
|
||||
# Strategy 1 — the property's own `location` wrapper (authoritative). Take
|
||||
# Strategy 1: the property's own `location` wrapper (authoritative). Take
|
||||
# the first match (the primary listing precedes any comparables in the
|
||||
# flight stream), but prefer one whose outcode matches the searched outcode.
|
||||
first_location = None
|
||||
|
|
@ -1269,7 +1269,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
|
|||
if first_location is not None:
|
||||
return attach_full_address(first_location)
|
||||
|
||||
# Strategy 2 — the `address` map-widget twin (same coordinates, backup).
|
||||
# Strategy 2: the `address` map-widget twin (same coordinates, backup).
|
||||
for match in _DETAIL_ADDRESS_RE.finditer(buf):
|
||||
full_address = match.group(1) or None
|
||||
body = match.group(2)
|
||||
|
|
@ -1284,7 +1284,7 @@ def parse_detail_geo(html: str, search_outcode: str | None = None) -> dict | Non
|
|||
|
||||
|
||||
def _detail_cache_key(listing_url: str) -> str:
|
||||
"""Cache key for a listing detail page — its numeric id when present."""
|
||||
"""Cache key for a listing detail page: its numeric id when present."""
|
||||
id_match = _LISTING_ID_RE.search(listing_url)
|
||||
return id_match.group(1) if id_match else listing_url
|
||||
|
||||
|
|
@ -1388,7 +1388,7 @@ def transform_property(
|
|||
location comes from the listing's detail page (see ``parse_detail_geo`` /
|
||||
``_fetch_listing_detail``), passed in as ``detail``. When detail-page
|
||||
coordinates are available we resolve the nearest postcode via the spatial
|
||||
index — mirroring rightmove/onthemarket — and only fall back to the coarse
|
||||
index (mirroring rightmove/onthemarket) and only fall back to the coarse
|
||||
outcode centroid when no detail location could be obtained."""
|
||||
price = parse_int_value(raw.get("price")) or 0
|
||||
|
||||
|
|
@ -1563,7 +1563,7 @@ def search_outcode(
|
|||
if detail_budget_seconds is not None:
|
||||
detail_deadline = time.monotonic() + detail_budget_seconds
|
||||
|
||||
# Always try extraction even if result count is 0 — the count regex may
|
||||
# Always try extraction even if result count is 0: the count regex may
|
||||
# not match Zoopla's current text format, but listings may still be in DOM
|
||||
raw_listings = _paginate(
|
||||
page,
|
||||
|
|
@ -1577,7 +1577,7 @@ def search_outcode(
|
|||
if not raw_listings:
|
||||
if total_results > 0:
|
||||
log.debug(
|
||||
"Zoopla %s %s: page claims %d results but extraction found 0 — "
|
||||
"Zoopla %s %s: page claims %d results but extraction found 0; "
|
||||
"DOM selectors may need updating",
|
||||
outcode, "BUY", total_results,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue