130 lines
4 KiB
Python
130 lines
4 KiB
Python
"""Tests for cooperative Ctrl+C shutdown across the scrapers."""
|
|
|
|
import signal
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
import http_client
|
|
import scraper
|
|
import shutdown
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_stop_flag():
|
|
"""Every test starts (and leaves) the global stop flag cleared."""
|
|
shutdown.reset()
|
|
yield
|
|
shutdown.reset()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The flag primitive
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_request_and_reset_toggle_the_flag():
|
|
assert shutdown.stop_requested() is False
|
|
shutdown.request_stop()
|
|
assert shutdown.stop_requested() is True
|
|
shutdown.reset()
|
|
assert shutdown.stop_requested() is False
|
|
|
|
|
|
def test_sleep_returns_immediately_once_stop_requested():
|
|
shutdown.request_stop()
|
|
started = time.monotonic()
|
|
shutdown.sleep(30) # would hang for 30s without the early wake
|
|
assert time.monotonic() - started < 1.0
|
|
|
|
|
|
def test_sleep_wakes_when_stop_requested_from_another_thread():
|
|
def _trip():
|
|
time.sleep(0.05)
|
|
shutdown.request_stop()
|
|
|
|
waker = threading.Thread(target=_trip)
|
|
waker.start()
|
|
started = time.monotonic()
|
|
shutdown.sleep(30)
|
|
waker.join()
|
|
assert time.monotonic() - started < 1.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Signal handlers: first signal requests stop, second forces a hard exit
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_signal_handler_requests_stop_then_escalates():
|
|
previous = signal.getsignal(signal.SIGINT)
|
|
try:
|
|
shutdown.install_signal_handlers()
|
|
handler = signal.getsignal(signal.SIGINT)
|
|
assert callable(handler)
|
|
|
|
# First Ctrl+C: graceful stop requested, no exception.
|
|
handler(signal.SIGINT, None)
|
|
assert shutdown.stop_requested() is True
|
|
|
|
# Second Ctrl+C: escalate to a hard interrupt.
|
|
with pytest.raises(KeyboardInterrupt):
|
|
handler(signal.SIGINT, None)
|
|
finally:
|
|
signal.signal(signal.SIGINT, previous)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# HTTP layer bails out instead of issuing more requests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_fetch_with_retry_returns_none_without_touching_client():
|
|
class _ExplodingClient:
|
|
def get(self, *_a, **_k): # pragma: no cover - must never be called
|
|
raise AssertionError("no request should be made after shutdown")
|
|
|
|
shutdown.request_stop()
|
|
assert http_client.fetch_with_retry(_ExplodingClient(), "http://x") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# A source loop stops at the next outcode boundary
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _londonish_prop(outcode):
|
|
return {"id": outcode, "Postcode": f"{outcode} 0HD", "price": 100000, "Bedrooms": 2}
|
|
|
|
|
|
def test_scrape_rightmove_stops_after_signal(monkeypatch):
|
|
visited = []
|
|
|
|
class _DummyClient:
|
|
def close(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(scraper, "make_client", lambda: _DummyClient())
|
|
monkeypatch.setattr(scraper, "resolve_outcode_id", lambda _client, oc: oc)
|
|
monkeypatch.setattr(scraper, "DELAY_BETWEEN_OUTCODES", 0)
|
|
|
|
def fake_search(_client, _oid, outcode, *_a, **_k):
|
|
visited.append(outcode)
|
|
# Operator hits Ctrl+C while the first outcode is being processed.
|
|
shutdown.request_stop()
|
|
return [_londonish_prop(outcode)]
|
|
|
|
monkeypatch.setattr(scraper, "rightmove_search_outcode", fake_search)
|
|
|
|
results = {source: [] for source in scraper.SOURCE_ORDER}
|
|
errors: list[str] = []
|
|
scraper._scrape_rightmove(
|
|
["SW9", "E1", "BR1"], pc_index=None, results=results,
|
|
errors=errors, max_properties_per_source=None,
|
|
)
|
|
|
|
# Only the first outcode ran; the loop bailed before the next boundary.
|
|
assert visited == ["SW9"]
|
|
assert len(results["rightmove"]) == 1
|
|
assert errors == []
|