150 lines
4.9 KiB
Python
150 lines
4.9 KiB
Python
"""Tests for the orchestrator's provider parallelism and cache persistence."""
|
|
|
|
import threading
|
|
|
|
import onthemarket
|
|
import rightmove
|
|
import scraper
|
|
import zoopla
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _run_sources: Zoopla inline, others in threads, failures isolated
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_run_sources_runs_every_source_and_isolates_failures():
|
|
order = []
|
|
lock = threading.Lock()
|
|
|
|
def rm():
|
|
with lock:
|
|
order.append("rm")
|
|
|
|
def otm():
|
|
with lock:
|
|
order.append("otm")
|
|
raise RuntimeError("otm boom")
|
|
|
|
def zoo():
|
|
with lock:
|
|
order.append("zoo")
|
|
|
|
errors: list[str] = []
|
|
scraper._run_sources(zoo, [("rightmove", rm), ("onthemarket", otm)], errors)
|
|
|
|
assert set(order) == {"rm", "otm", "zoo"}
|
|
# The failing source is recorded but did not stop the others.
|
|
assert any("onthemarket" in e and "boom" in e for e in errors)
|
|
assert not any("rightmove" in e for e in errors)
|
|
|
|
|
|
def test_run_sources_records_zoopla_failure():
|
|
errors: list[str] = []
|
|
|
|
def zoo():
|
|
raise ValueError("zoo down")
|
|
|
|
scraper._run_sources(zoo, [], errors)
|
|
assert any("zoopla" in e and "zoo down" in e for e in errors)
|
|
|
|
|
|
def test_run_sources_handles_no_background_runners():
|
|
ran = []
|
|
errors: list[str] = []
|
|
scraper._run_sources(lambda: ran.append("z"), [], errors)
|
|
assert ran == ["z"]
|
|
assert errors == []
|
|
|
|
|
|
def test_run_sources_handles_zoopla_absent():
|
|
ran = []
|
|
errors: list[str] = []
|
|
scraper._run_sources(None, [("rightmove", lambda: ran.append("rm"))], errors)
|
|
assert ran == ["rm"]
|
|
assert errors == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Persistent cache wiring
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_seed_and_save_detail_caches_round_trip(tmp_path):
|
|
rightmove._detail_postcode_cache.clear()
|
|
rightmove._detail_postcode_cache.update({"111": "SW9 0HD", "222": None})
|
|
|
|
scraper._save_detail_caches(["rightmove"], tmp_path)
|
|
assert (tmp_path / "rightmove.json").exists()
|
|
|
|
rightmove._detail_postcode_cache.clear()
|
|
scraper._seed_detail_caches(["rightmove"], tmp_path)
|
|
assert rightmove._detail_postcode_cache == {"111": "SW9 0HD", "222": None}
|
|
rightmove._detail_postcode_cache.clear()
|
|
|
|
|
|
def test_seed_detail_caches_tolerates_missing_files(tmp_path):
|
|
rightmove._detail_postcode_cache.clear()
|
|
# No file written yet: seeding must not raise and must leave cache empty.
|
|
scraper._seed_detail_caches(["rightmove"], tmp_path)
|
|
assert rightmove._detail_postcode_cache == {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# run_scrape: full orchestration wiring (sources stubbed, no network)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _fake_prop(pid, postcode):
|
|
# Minimal shape: only the fields _merge_properties / _dedup_key read.
|
|
return {"id": pid, "Postcode": postcode, "price": 100000, "Bedrooms": 2}
|
|
|
|
|
|
def test_run_scrape_runs_all_sources_merges_and_persists_caches(tmp_path, monkeypatch):
|
|
for module in (rightmove, onthemarket, zoopla):
|
|
module.detail_cache_snapshot() # ensure hooks exist
|
|
rightmove._detail_postcode_cache.clear()
|
|
onthemarket._detail_postcode_cache.clear()
|
|
zoopla._detail_cache.clear()
|
|
|
|
def fake_rm(outcodes, pc_index, results, errors, cap):
|
|
results["rightmove"].append(_fake_prop("rm_1", "SW9 0HD"))
|
|
|
|
def fake_otm(outcodes, pc_index, results, errors, cap):
|
|
results["onthemarket"].append(_fake_prop("otm_1", "E1 6AN"))
|
|
|
|
def fake_zoo(outcodes, pc_index, pc_coords, results, errors, cap):
|
|
results["zoopla"].append(_fake_prop("z_1", "BR1 2AB"))
|
|
|
|
monkeypatch.setattr(scraper, "_scrape_rightmove", fake_rm)
|
|
monkeypatch.setattr(scraper, "_scrape_onthemarket", fake_otm)
|
|
monkeypatch.setattr(scraper, "_scrape_zoopla", fake_zoo)
|
|
monkeypatch.setattr(scraper, "build_postcode_coords", lambda: {})
|
|
|
|
written = {}
|
|
monkeypatch.setattr(
|
|
scraper,
|
|
"write_parquet",
|
|
lambda props, path: written.update(count=len(props), path=path),
|
|
)
|
|
|
|
result = scraper.run_scrape(
|
|
["SW9", "E1", "BR1"],
|
|
pc_index=None,
|
|
pc_coords=None,
|
|
sources=["rightmove", "onthemarket", "zoopla"],
|
|
output_dir=tmp_path,
|
|
max_properties_per_source=None,
|
|
)
|
|
|
|
# All three sources ran and their listings were merged + written.
|
|
assert written["count"] == 3
|
|
assert result["counts"]["total"] == 3
|
|
assert result["errors"] == []
|
|
# Persistent cache files were created for every selected source.
|
|
for source in ("rightmove", "onthemarket", "zoopla"):
|
|
assert (tmp_path / "detail_cache" / f"{source}.json").exists()
|
|
|
|
rightmove._detail_postcode_cache.clear()
|
|
onthemarket._detail_postcode_cache.clear()
|
|
zoopla._detail_cache.clear()
|