Small fixes

finder/test_scraper_concurrency.py

@@ -0,0 +1,150 @@
+"""Tests for the orchestrator's provider parallelism and cache persistence."""
+
+import threading
+
+import onthemarket
+import rightmove
+import scraper
+import zoopla
+
+
+# ---------------------------------------------------------------------------
+# _run_sources — Zoopla inline, others in threads, failures isolated
+# ---------------------------------------------------------------------------
+
+
+def test_run_sources_runs_every_source_and_isolates_failures():
+    order = []
+    lock = threading.Lock()
+
+    def rm():
+        with lock:
+            order.append("rm")
+
+    def otm():
+        with lock:
+            order.append("otm")
+        raise RuntimeError("otm boom")
+
+    def zoo():
+        with lock:
+            order.append("zoo")
+
+    errors: list[str] = []
+    scraper._run_sources(zoo, [("rightmove", rm), ("onthemarket", otm)], errors)
+
+    assert set(order) == {"rm", "otm", "zoo"}
+    # The failing source is recorded but did not stop the others.
+    assert any("onthemarket" in e and "boom" in e for e in errors)
+    assert not any("rightmove" in e for e in errors)
+
+
+def test_run_sources_records_zoopla_failure():
+    errors: list[str] = []
+
+    def zoo():
+        raise ValueError("zoo down")
+
+    scraper._run_sources(zoo, [], errors)
+    assert any("zoopla" in e and "zoo down" in e for e in errors)
+
+
+def test_run_sources_handles_no_background_runners():
+    ran = []
+    errors: list[str] = []
+    scraper._run_sources(lambda: ran.append("z"), [], errors)
+    assert ran == ["z"]
+    assert errors == []
+
+
+def test_run_sources_handles_zoopla_absent():
+    ran = []
+    errors: list[str] = []
+    scraper._run_sources(None, [("rightmove", lambda: ran.append("rm"))], errors)
+    assert ran == ["rm"]
+    assert errors == []
+
+
+# ---------------------------------------------------------------------------
+# Persistent cache wiring
+# ---------------------------------------------------------------------------
+
+
+def test_seed_and_save_detail_caches_round_trip(tmp_path):
+    rightmove._detail_postcode_cache.clear()
+    rightmove._detail_postcode_cache.update({"111": "SW9 0HD", "222": None})
+
+    scraper._save_detail_caches(["rightmove"], tmp_path)
+    assert (tmp_path / "rightmove.json").exists()
+
+    rightmove._detail_postcode_cache.clear()
+    scraper._seed_detail_caches(["rightmove"], tmp_path)
+    assert rightmove._detail_postcode_cache == {"111": "SW9 0HD", "222": None}
+    rightmove._detail_postcode_cache.clear()
+
+
+def test_seed_detail_caches_tolerates_missing_files(tmp_path):
+    rightmove._detail_postcode_cache.clear()
+    # No file written yet — seeding must not raise and must leave cache empty.
+    scraper._seed_detail_caches(["rightmove"], tmp_path)
+    assert rightmove._detail_postcode_cache == {}
+
+
+# ---------------------------------------------------------------------------
+# run_scrape — full orchestration wiring (sources stubbed, no network)
+# ---------------------------------------------------------------------------
+
+
+def _fake_prop(pid, postcode):
+    # Minimal shape: only the fields _merge_properties / _dedup_key read.
+    return {"id": pid, "Postcode": postcode, "price": 100000, "Bedrooms": 2}
+
+
+def test_run_scrape_runs_all_sources_merges_and_persists_caches(tmp_path, monkeypatch):
+    for module in (rightmove, onthemarket, zoopla):
+        module.detail_cache_snapshot()  # ensure hooks exist
+    rightmove._detail_postcode_cache.clear()
+    onthemarket._detail_postcode_cache.clear()
+    zoopla._detail_cache.clear()
+
+    def fake_rm(outcodes, pc_index, results, errors, cap):
+        results["rightmove"].append(_fake_prop("rm_1", "SW9 0HD"))
+
+    def fake_otm(outcodes, pc_index, results, errors, cap):
+        results["onthemarket"].append(_fake_prop("otm_1", "E1 6AN"))
+
+    def fake_zoo(outcodes, pc_index, pc_coords, results, errors, cap):
+        results["zoopla"].append(_fake_prop("z_1", "BR1 2AB"))
+
+    monkeypatch.setattr(scraper, "_scrape_rightmove", fake_rm)
+    monkeypatch.setattr(scraper, "_scrape_onthemarket", fake_otm)
+    monkeypatch.setattr(scraper, "_scrape_zoopla", fake_zoo)
+    monkeypatch.setattr(scraper, "build_postcode_coords", lambda: {})
+
+    written = {}
+    monkeypatch.setattr(
+        scraper,
+        "write_parquet",
+        lambda props, path: written.update(count=len(props), path=path),
+    )
+
+    result = scraper.run_scrape(
+        ["SW9", "E1", "BR1"],
+        pc_index=None,
+        pc_coords=None,
+        sources=["rightmove", "onthemarket", "zoopla"],
+        output_dir=tmp_path,
+        max_properties_per_source=None,
+    )
+
+    # All three sources ran and their listings were merged + written.
+    assert written["count"] == 3
+    assert result["counts"]["total"] == 3
+    assert result["errors"] == []
+    # Persistent cache files were created for every selected source.
+    for source in ("rightmove", "onthemarket", "zoopla"):
+        assert (tmp_path / "detail_cache" / f"{source}.json").exists()
+
+    rightmove._detail_postcode_cache.clear()
+    onthemarket._detail_postcode_cache.clear()
+    zoopla._detail_cache.clear()