ok

finder/http_client.py

@@ -23,7 +23,7 @@ class RateLimiter:
     Detail-page fetches run concurrently across many worker threads (and across
     providers), but a single shared limiter caps their COMBINED rate so the VPN
     egress IP stays polite. Each ``acquire()`` reserves the next free time slot
-    under a lock, then sleeps (outside the lock) until that slot — so N threads
+    under a lock, then sleeps (outside the lock) until that slot, so N threads
     calling concurrently are spaced ``1/rate_per_second`` apart rather than all
     firing at once. ``rate_per_second <= 0`` disables limiting."""
 

finder/postcode_cache.py

@@ -2,7 +2,7 @@
 
 Each portal recovers a listing's true postcode (Rightmove/OnTheMarket) or full
 geo dict (Zoopla) from its detail page. That value never changes for a given
-listing id, yet the in-memory caches are discarded at the end of every run — so
+listing id, yet the in-memory caches are discarded at the end of every run, so
 each run re-fetches every listing's detail page from scratch. Persisting the
 cache to disk means a steady-state run only fetches NEWLY-appeared listings,
 typically a small fraction of the market, which is the single biggest saving
@@ -27,7 +27,7 @@ log = logging.getLogger("rightmove")
 def load_cache(path: str | Path) -> dict:
     """Load a persisted detail cache. Returns ``{}`` when absent or unreadable.
 
-    A corrupt or non-object file is treated as empty rather than fatal — a bad
+    A corrupt or non-object file is treated as empty rather than fatal: a bad
     cache must never block a scrape; the worst case is re-fetching details."""
     p = Path(path)
     if not p.exists():

finder/rightmove.py

@@ -36,7 +36,7 @@ _MAX_INDEX = 1008
 # ---------------------------------------------------------------------------
 #
 # The search API (_paginate) only returns an outcode-level `displayAddress`
-# (e.g. "Akerman Road, Brixton, London, SW9") — never the full postcode. Each
+# (e.g. "Akerman Road, Brixton, London, SW9"), never the full postcode. Each
 # listing's detail page, however, embeds the property's OWN full postcode in a
 # `window.__PAGE_MODEL` script as `propertyData.address.{outcode, incode}`
 # (e.g. outcode "SW9" + incode "0HD" → "SW9 0HD"), independently corroborated by
@@ -51,8 +51,8 @@ _MAX_INDEX = 1008
 # __PAGE_MODEL is a "devalue"-style flattened object graph: its `data` field is
 # a JSON STRING holding a flat array where every integer inside a container is
 # an index reference into that same array (so the graph can dedupe). We
-# brace-match the (large, deeply-nested) object literal — a non-greedy regex
-# cannot — then rehydrate the reference graph before reading the address.
+# brace-match the (large, deeply-nested) object literal (a non-greedy regex
+# cannot), then rehydrate the reference graph before reading the address.
 
 _PAGE_MODEL_RE = re.compile(r"window\.__PAGE_MODEL\s*=\s*")
 
@@ -128,7 +128,7 @@ def parse_detail_postcode(html: str) -> str | None:
     Pure and network-free so it is unit-testable: callers pass the page HTML.
     Reads ``propertyData.address.outcode`` + ``.incode`` from window.__PAGE_MODEL
     and returns a normalised full postcode (e.g. "SW9 0HD"), or None when the
-    page has no parseable address (the property location wrapper can be empty —
+    page has no parseable address (the property location wrapper can be empty;
     the caller then keeps the coordinate fallback). The returned outcode is
     re-validated against the joined postcode so a malformed incode is dropped.
     """
@@ -193,7 +193,7 @@ def _fetch_detail_postcode(client: httpx.Client, property_id: str) -> str | None
     """GET a listing detail page and return its true full postcode (or None).
 
     Results (including failures) are cached by listing id. The detail page is a
-    plain HTML GET — no Cloudflare, unlike Zoopla — so a single httpx call
+    plain HTML GET (no Cloudflare, unlike Zoopla), so a single httpx call
     suffices; any error degrades gracefully to the coordinate fallback. Safe to
     call concurrently: distinct listing ids write distinct cache keys, and the
     shared RATE_LIMITER spaces the GETs."""
@@ -245,7 +245,7 @@ def _needs_detail_fetch(prop: dict) -> bool:
     Skips listings the search already pins precisely: an "ACCURATE_POINT"
     ``pinType`` means rooftop-exact coordinates, so the coordinate-nearest
     postcode is trustworthy and the detail page would only confirm it. Listings
-    with an approximate pin — or no ``pinType`` field at all — still get fetched,
+    with an approximate pin (or no ``pinType`` field at all) still get fetched,
     so this degrades safely to the previous behaviour when the search payload
     omits ``pinType``."""
     if not RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS:
@@ -262,8 +262,8 @@ def _prime_detail_postcodes(
 ) -> None:
     """Fill ``_detail_postcode_cache`` for the listings that need a detail page.
 
-    Picks the fresh (uncached, not-skipped) listings — up to ``detail_cap`` per
-    outcode — then fetches their detail pages CONCURRENTLY, bounded by
+    Picks the fresh (uncached, not-skipped) listings, up to ``detail_cap`` per
+    outcode, then fetches their detail pages CONCURRENTLY, bounded by
     ``DETAIL_FETCH_CONCURRENCY`` (the shared RATE_LIMITER keeps the combined
     request rate polite). Cached listings cost neither a slot nor a GET. The
     worklist is deduplicated, so distinct ids write distinct cache keys and the
@@ -305,8 +305,8 @@ def _collect_search_props(
 ) -> tuple[list[dict], int]:
     """Paginate the search API for one outcode+channel, collecting raw results.
 
-    Returns ``(raw_props, result_count)``. Pagination stays serial — each page
-    reveals the next — but is cheap relative to detail fetching, and the
+    Returns ``(raw_props, result_count)``. Pagination stays serial (each page
+    reveals the next) but is cheap relative to detail fetching, and the
     RATE_LIMITER spaces the page GETs. Collection stops at ``max_properties`` raw
     listings, the end of results, or Rightmove's ``_MAX_INDEX`` page cap."""
     raw_props: list[dict] = []
@@ -324,6 +324,8 @@ def _collect_search_props(
             "channel": channel_cfg["channel"],
             "transactionType": channel_cfg["transactionType"],
         }
+        # Optional per-channel filters, e.g. `mustHave=newHome` for the new-homes pass.
+        params.update(channel_cfg.get("extra_params", {}))
         data = fetch_with_retry(client, SEARCH_URL, params)
         if not data:
             log.warning(
@@ -372,7 +374,7 @@ def _paginate(
 ) -> tuple[list[dict], int]:
     """Collect search results, recover true postcodes, and transform them.
 
-    Search pages are paginated serially; then — when ``fetch_details`` is set —
+    Search pages are paginated serially; then, when ``fetch_details`` is set,
     up to ``detail_cap`` listings per outcode have their detail page fetched
     CONCURRENTLY for the property's TRUE full postcode (see
     ``parse_detail_postcode``), with listings the search already pins precisely

finder/scraper.py

@@ -14,6 +14,7 @@ import polars as pl
 from constants import (
     ARCGIS_PATH,
     CHANNELS,
+    NEW_HOMES_CHANNEL,
     DATA_DIR,
     DELAY_BETWEEN_OUTCODES,
     LONDON_OUTCODE_PREFIXES,
@@ -368,6 +369,34 @@ def _scrape_rightmove(
             except Exception as exc:
                 _record_error(errors, "rightmove", outcode, exc)
 
+            # Second pass: new-build developments (mustHave=newHome). These can
+            # rank low in the default resale sort, so a dedicated pass ensures
+            # they are captured; transform_property stamps their URL as RES_NEW.
+            # Overlap with the resale pass is removed by id in _merge_properties.
+            # Skipped if a stop was requested mid-outcode (don't start a new pass).
+            remaining = _source_remaining(
+                results, "rightmove", max_properties_per_source
+            )
+            if not shutdown.stop_requested() and remaining != 0:
+                try:
+                    new_props = rightmove_search_outcode(
+                        client,
+                        outcode_id,
+                        outcode,
+                        NEW_HOMES_CHANNEL,
+                        pc_index,
+                        max_properties=remaining,
+                    )
+                    added_new = _store_properties(
+                        results,
+                        "rightmove",
+                        new_props,
+                        max_properties_per_source,
+                    )
+                    log.info("Rightmove %s new-homes: +%d", outcode, added_new)
+                except Exception as exc:
+                    _record_error(errors, "rightmove", outcode, exc)
+
             shutdown.sleep(DELAY_BETWEEN_OUTCODES)
     finally:
         client.close()
@@ -378,7 +407,7 @@ class OutcodeTimeout(BaseException):
 
     Inherits BaseException (not Exception) so the SIGALRM-triggered raise can't
     be silently swallowed by any of the broad `except Exception:` handlers
-    inside zoopla.py — the signal may fire at any bytecode boundary, including
+    inside zoopla.py: the signal may fire at any bytecode boundary, including
     inside those handlers."""
 
 
@@ -431,7 +460,7 @@ def _wall_clock_timeout(seconds: int, label: str):
 
     Interrupts a hung Playwright IPC by delivering SIGALRM to the main thread;
     socket waits return EINTR and the handler raises into the caller. The
-    browser is presumed unhealthy afterwards — caller must relaunch it."""
+    browser is presumed unhealthy afterwards. Caller must relaunch it."""
     if seconds <= 0:
         yield
         return

finder/shutdown.py

@@ -4,8 +4,8 @@ A single :class:`threading.Event` is set the first time the process receives
 SIGINT (Ctrl+C) or SIGTERM. Every scrape loop polls :func:`stop_requested` at
 its outcode/page boundaries and every blocking delay goes through :func:`sleep`,
 which wakes the instant a stop is requested. So Ctrl+C makes each source stop
-*starting* new work and unwind through its normal ``finally`` blocks — detail
-caches are persisted and whatever has been collected so far is still written —
+*starting* new work and unwind through its normal ``finally`` blocks (detail
+caches are persisted and whatever has been collected so far is still written)
 instead of hanging until the worker threads happen to finish (the orchestrator's
 ``ThreadPoolExecutor`` used to block the exit waiting on them) or losing the run
 outright.
@@ -34,7 +34,7 @@ def request_stop() -> None:
 
 
 def reset() -> None:
-    """Clear the flag — for tests and repeated in-process runs."""
+    """Clear the flag (for tests and repeated in-process runs)."""
     _STOP.clear()
 
 

finder/storage.py

@@ -16,7 +16,7 @@ def write_parquet(properties: list[dict], path: Path) -> None:
         log.warning("No properties to write to %s", path)
         return
 
-    # Sanitize bedroom/bathroom counts — values above MAX_BEDROOMS are
+    # Sanitize bedroom/bathroom counts: values above MAX_BEDROOMS are
     # almost certainly prices or other numeric fields mis-parsed as bedrooms.
     bad_count = 0
     for p in properties:
@@ -91,7 +91,7 @@ def write_parquet(properties: list[dict], path: Path) -> None:
         else:
             listing_dates.append(None)
 
-    # Zero prices indicate parsing failures or POA/auction listings — treat as null
+    # Zero prices indicate parsing failures or POA/auction listings: treat as null
     asking_prices = [p["price"] if p["price"] > 0 else None for p in properties]
     listing_statuses = ["For sale"] * len(properties)
 

finder/test_http_client.py

@@ -73,7 +73,7 @@ def test_zero_rate_disables_limiting(monkeypatch):
 def test_concurrent_acquires_are_all_spaced(monkeypatch):
     # Real clock, tiny rate: N threads hitting acquire() at once must be
     # serialised so the total wall time is at least (N-1) * interval.
-    rl = RateLimiter(200)  # 5ms interval — fast but measurable
+    rl = RateLimiter(200)  # 5ms interval, fast but measurable
     barrier = threading.Barrier(8)
 
     def worker():

finder/test_main.py

@@ -6,7 +6,7 @@ import main
 
 
 # ---------------------------------------------------------------------------
-# selected_sources — comma-separated --source values
+# selected_sources: comma-separated --source values
 # ---------------------------------------------------------------------------
 
 

finder/test_onthemarket.py

@@ -5,7 +5,7 @@ or None), so these tests use a trimmed but faithful copy of a real OnTheMarket
 detail page's `__NEXT_DATA__` payload. The fixture mirrors the live structure:
 the property's own postcode lives in the analytics dataLayer
 (`props.initialReduxState.metadata.dataLayer.postcode`) while the agent's office
-postcode sits separately under `…property.agent.postcode` — the trap we must not
+postcode sits separately under `…property.agent.postcode`, the trap we must not
 fall into.
 """
 
@@ -49,7 +49,7 @@ def _detail_html(
                 "property": {
                     "displayAddress": "Padfield Road, London, SE5",
                     "location": {"lon": -0.100233, "lat": 51.466129},
-                    # The agent block carries the AGENT'S office postcode — the
+                    # The agent block carries the AGENT'S office postcode, the
                     # trap. parse_detail_postcode must not return this.
                     "agent": {
                         "address": "29 Denmark Hill, Camberwell\nLondon\nSE5 8RS",
@@ -125,7 +125,7 @@ def test_parse_handles_missing_datalayer():
 
 
 # ---------------------------------------------------------------------------
-# transform_property — detail postcode wiring + trust rule
+# transform_property: detail postcode wiring + trust rule
 # ---------------------------------------------------------------------------
 
 
@@ -176,7 +176,7 @@ def test_transform_without_detail_postcode_uses_coordinates():
 def test_transform_detail_postcode_via_search_address_outcode():
     # When the card address already carries a full postcode that agrees with the
     # coordinates, the existing "address" source still wins absent a detail
-    # postcode — detail recovery never regresses that path.
+    # postcode. Detail recovery never regresses that path.
     raw = dict(_RAW_LISTING, address="Padfield Road, London, SE5 1AA")
     index = _StubIndex("SE5 1AA")
     out = transform_property(raw, index, detail_postcode=None)

finder/test_rightmove_channels.py

@@ -0,0 +1,33 @@
+"""Tests for the new-homes search pass (mustHave=newHome) channel wiring."""
+
+import rightmove
+from constants import CHANNELS, NEW_HOMES_CHANNEL
+from rightmove import _collect_search_props
+
+
+def _capture_params(monkeypatch):
+    captured: list[dict] = []
+
+    def fake_fetch(client, url, params):
+        captured.append(dict(params))
+        return {"properties": [], "resultCount": "0"}
+
+    monkeypatch.setattr(rightmove, "fetch_with_retry", fake_fetch)
+    return captured
+
+
+def test_new_homes_channel_sends_must_have_new_home(monkeypatch):
+    captured = _capture_params(monkeypatch)
+    _collect_search_props(None, "749", "E14", NEW_HOMES_CHANNEL)
+    assert captured, "no search request was issued"
+    assert captured[0].get("mustHave") == "newHome"
+    # New homes are still requested on the BUY channel; only the filter differs.
+    assert captured[0]["channel"] == "BUY"
+    assert captured[0]["transactionType"] == "BUY"
+
+
+def test_resale_channel_sends_no_extra_filters(monkeypatch):
+    captured = _capture_params(monkeypatch)
+    _collect_search_props(None, "749", "E14", CHANNELS[0])
+    assert captured, "no search request was issued"
+    assert "mustHave" not in captured[0]