wip

finder/onthemarket.py

@@ -1,4 +1,4 @@
-"""OnTheMarket (onthemarket.com) scraper — sale properties.
+"""OnTheMarket (onthemarket.com) scraper: sale properties.
 
 OnTheMarket serves a Next.js app with the full search-results payload embedded
 as JSON in a `__NEXT_DATA__` script tag. No JS execution or browser needed:
@@ -15,19 +15,19 @@ Postcodes
 ---------
 The search card exposes only an *outcode*-level address (e.g. "Padfield Road,
 London, SE5") and a map pin, so the old behaviour derived the postcode from the
-nearest postcode to that pin — a guess that frequently lands on a neighbouring
+nearest postcode to that pin, a guess that frequently lands on a neighbouring
 unit (the pin can sit on the wrong side of a street boundary).
 
 Each *detail* page (`/details/{id}/`) is a plain HTTPS GET whose `__NEXT_DATA__`
 embeds the property's analytics dataLayer at
 `props.initialReduxState.metadata.dataLayer`, which carries the property's own
 `postcode` (full unit postcode, e.g. "SE5 9AA") keyed to this listing by
-`property-id`. Crucially this is NOT the agent's office postcode — that lives
+`property-id`. Crucially this is NOT the agent's office postcode. That lives
 separately at `…property.agent.postcode` ("SE5 8RS" for the same listing) and
 is the classic trap when blindly scanning the page for a postcode. We read the
 dataLayer postcode, verify `property-id` matches the listing, and accept it only
 when its outcode agrees with the coordinate-nearest postcode (via
-``resolve_listing_postcode``) — exactly the trust rule the other scrapers use.
+``resolve_listing_postcode``), exactly the trust rule the other scrapers use.
 Measured over a sample of real listings this yields a trustworthy, usually
 exact-unit postcode for ~11/12 listings; the rest safely fall back to the
 coordinate-nearest postcode.
@@ -68,8 +68,8 @@ from transform import (
 log = logging.getLogger("rightmove")
 
 # Detail-page postcode recovery (see module docstring). When enabled, each
-# listing's detail page is fetched so its analytics dataLayer postcode — the
-# property's own full unit postcode — can replace the coordinate-nearest guess.
+# listing's detail page is fetched so its analytics dataLayer postcode (the
+# property's own full unit postcode) can replace the coordinate-nearest guess.
 # Bounded per outcode so a large outcode can't balloon into unbounded extra
 # HTTPS GETs. Kept at parity with the Rightmove/Zoopla detail caps (400) so a
 # typical outcode's listings all get their real postcode rather than a
@@ -145,7 +145,7 @@ def _fetch_page_json(client: httpx.Client, outcode: str, page_num: int) -> dict
 
         if 300 <= resp.status_code < 400:
             log.debug(
-                "OnTheMarket %s page %d redirected (%d) — end of results",
+                "OnTheMarket %s page %d redirected (%d): end of results",
                 outcode, page_num, resp.status_code,
             )
             return None
@@ -189,7 +189,7 @@ def parse_detail_postcode(html: str, listing_id: str | None = None) -> str | Non
     ``props.initialReduxState.metadata.dataLayer.postcode`` and is the
     property's own unit postcode (e.g. "SE5 9AA"). It is deliberately NOT the
     agent's office postcode, which sits separately at
-    ``…property.agent.postcode`` — the trap when scanning a detail page for "a"
+    ``…property.agent.postcode``, the trap when scanning a detail page for "a"
     postcode. When ``listing_id`` is given, the dataLayer's ``property-id`` must
     match it, guaranteeing we read this listing's postcode and not a stray one.
 
@@ -235,7 +235,7 @@ def _fetch_detail_postcode(
 
     Results (including failures) are cached by listing id so a listing that
     reappears across overlapping outcode searches is fetched at most once. Plain
-    HTTPS GET — OnTheMarket detail pages have no Cloudflare challenge. Network /
+    HTTPS GET: OnTheMarket detail pages have no Cloudflare challenge. Network /
     parse errors degrade gracefully to None so the caller falls back to the
     coordinate-nearest postcode. Safe to call concurrently: distinct listing ids
     write distinct cache keys, and the shared RATE_LIMITER spaces the GETs.
@@ -452,7 +452,7 @@ def _prime_detail_postcodes(
 ) -> None:
     """Fill ``_detail_postcode_cache`` for the listings that need a detail page.
 
-    Picks the fresh (uncached) listings — up to ``detail_cap`` per outcode — then
+    Picks the fresh (uncached) listings, up to ``detail_cap`` per outcode, then
     fetches their detail pages CONCURRENTLY, bounded by
     ``DETAIL_FETCH_CONCURRENCY`` (the shared RATE_LIMITER keeps the combined
     request rate polite). Cached listings cost neither a slot nor a GET. The