69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
"""Process-wide cooperative shutdown for the scrapers.
|
|
|
|
A single :class:`threading.Event` is set the first time the process receives
|
|
SIGINT (Ctrl+C) or SIGTERM. Every scrape loop polls :func:`stop_requested` at
|
|
its outcode/page boundaries and every blocking delay goes through :func:`sleep`,
|
|
which wakes the instant a stop is requested. So Ctrl+C makes each source stop
|
|
*starting* new work and unwind through its normal ``finally`` blocks — detail
|
|
caches are persisted and whatever has been collected so far is still written —
|
|
instead of hanging until the worker threads happen to finish (the orchestrator's
|
|
``ThreadPoolExecutor`` used to block the exit waiting on them) or losing the run
|
|
outright.
|
|
|
|
A second Ctrl+C escalates to a hard :class:`KeyboardInterrupt` for an impatient
|
|
operator. SIGALRM is deliberately left untouched: Zoopla's per-outcode
|
|
wall-clock guard owns it."""
|
|
|
|
import logging
|
|
import signal
|
|
import threading
|
|
|
|
log = logging.getLogger("finder")
|
|
|
|
_STOP = threading.Event()
|
|
|
|
|
|
def stop_requested() -> bool:
|
|
"""True once a shutdown signal has been received."""
|
|
return _STOP.is_set()
|
|
|
|
|
|
def request_stop() -> None:
|
|
"""Ask every scrape loop to stop starting new work."""
|
|
_STOP.set()
|
|
|
|
|
|
def reset() -> None:
|
|
"""Clear the flag — for tests and repeated in-process runs."""
|
|
_STOP.clear()
|
|
|
|
|
|
def sleep(seconds: float) -> None:
|
|
"""Sleep that wakes immediately once a stop is requested.
|
|
|
|
Drop-in for ``time.sleep`` in the scrape loops, inter-outcode pauses and
|
|
retry backoffs so a pending Ctrl+C is never stuck behind a multi-second
|
|
delay."""
|
|
if seconds <= 0:
|
|
return
|
|
_STOP.wait(seconds)
|
|
|
|
|
|
def install_signal_handlers() -> None:
|
|
"""Route SIGINT/SIGTERM to :func:`request_stop` (second signal = hard exit).
|
|
|
|
Signal handlers run in the main thread, so this must be called from there."""
|
|
|
|
def _handle(signum, _frame):
|
|
if _STOP.is_set():
|
|
# Operator pressed Ctrl+C twice: abandon the graceful unwind.
|
|
raise KeyboardInterrupt
|
|
log.warning(
|
|
"Signal %s received; finishing in-flight work and saving partial "
|
|
"results (press Ctrl+C again to force-quit)...",
|
|
signal.Signals(signum).name,
|
|
)
|
|
_STOP.set()
|
|
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(sig, _handle)
|