Cache postcodes
This commit is contained in:
parent
f59d01227b
commit
8e4c56bb0d
7 changed files with 502 additions and 141 deletions
|
|
@ -3,9 +3,11 @@ import os
|
|||
import re
|
||||
import signal
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from typing import Callable, Iterable
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
|
@ -21,8 +23,13 @@ from constants import (
|
|||
ZOOPLA_MAX_DETAILS_PER_OUTCODE,
|
||||
)
|
||||
|
||||
import onthemarket
|
||||
import rightmove
|
||||
import shutdown
|
||||
import zoopla
|
||||
from http_client import make_client
|
||||
from onthemarket import search_outcode as onthemarket_search_outcode
|
||||
from postcode_cache import load_cache, save_cache
|
||||
from rightmove import resolve_outcode_id
|
||||
from rightmove import search_outcode as rightmove_search_outcode
|
||||
from spatial import PostcodeSpatialIndex
|
||||
|
|
@ -38,6 +45,16 @@ SALE_CHANNEL = CHANNELS[0]
|
|||
LONDON_AREAS = sorted({prefix.upper() for prefix in LONDON_OUTCODE_PREFIXES})
|
||||
OUTCODE_RE = re.compile(r"^([A-Z]{1,2}\d[A-Z0-9]?)")
|
||||
|
||||
# Per-source modules exposing the persistent detail-cache hooks
|
||||
# (seed_detail_cache / detail_cache_snapshot). A listing's recovered postcode
|
||||
# never changes, so the cache is loaded before a run and dumped after it, letting
|
||||
# a recurring scrape skip the detail fetch for every listing it has already seen.
|
||||
_CACHE_MODULES = {
|
||||
"rightmove": rightmove,
|
||||
"onthemarket": onthemarket,
|
||||
"zoopla": zoopla,
|
||||
}
|
||||
|
||||
|
||||
def _arcgis_columns() -> tuple[str, str]:
|
||||
"""Return postcode and country column names for supported ARCGIS schemas."""
|
||||
|
|
@ -306,6 +323,9 @@ def _scrape_rightmove(
|
|||
client = make_client()
|
||||
try:
|
||||
for outcode in outcodes:
|
||||
if shutdown.stop_requested():
|
||||
log.info("Rightmove: shutdown requested, stopping early")
|
||||
return
|
||||
if _source_remaining(results, "rightmove", max_properties_per_source) == 0:
|
||||
log.info("Rightmove cap reached")
|
||||
return
|
||||
|
|
@ -314,12 +334,12 @@ def _scrape_rightmove(
|
|||
outcode_id = resolve_outcode_id(client, outcode)
|
||||
except Exception as exc:
|
||||
_record_error(errors, "rightmove", outcode, exc)
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
continue
|
||||
|
||||
if not outcode_id:
|
||||
log.debug("No Rightmove outcode ID for %s", outcode)
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
continue
|
||||
|
||||
remaining = _source_remaining(
|
||||
|
|
@ -348,7 +368,7 @@ def _scrape_rightmove(
|
|||
except Exception as exc:
|
||||
_record_error(errors, "rightmove", outcode, exc)
|
||||
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
|
@ -494,6 +514,9 @@ def _scrape_zoopla_flaresolverr(
|
|||
|
||||
try:
|
||||
for outcode in outcodes:
|
||||
if shutdown.stop_requested():
|
||||
log.info("Zoopla: shutdown requested, stopping early")
|
||||
return
|
||||
remaining = _source_remaining(results, "zoopla", max_properties_per_source)
|
||||
if remaining == 0:
|
||||
log.info("Zoopla cap reached")
|
||||
|
|
@ -511,7 +534,7 @@ def _scrape_zoopla_flaresolverr(
|
|||
log.info("Zoopla %s: +%d", outcode, added)
|
||||
except Exception as exc: # noqa: BLE001 - one outcode must not kill the run
|
||||
_record_error(errors, "zoopla", outcode, exc)
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
finally:
|
||||
session.__exit__(None, None, None)
|
||||
|
||||
|
|
@ -547,6 +570,9 @@ def _scrape_zoopla(
|
|||
|
||||
try:
|
||||
for outcode in outcodes:
|
||||
if shutdown.stop_requested():
|
||||
log.info("Zoopla: shutdown requested, stopping early")
|
||||
return
|
||||
if _source_remaining(results, "zoopla", max_properties_per_source) == 0:
|
||||
log.info("Zoopla cap reached")
|
||||
return
|
||||
|
|
@ -596,7 +622,7 @@ def _scrape_zoopla(
|
|||
_record_error(errors, "zoopla", outcode, relaunch_exc)
|
||||
return
|
||||
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
finally:
|
||||
if detail_page is not None:
|
||||
try:
|
||||
|
|
@ -616,6 +642,9 @@ def _scrape_onthemarket(
|
|||
client = make_client()
|
||||
try:
|
||||
for outcode in outcodes:
|
||||
if shutdown.stop_requested():
|
||||
log.info("OnTheMarket: shutdown requested, stopping early")
|
||||
return
|
||||
if (
|
||||
_source_remaining(results, "onthemarket", max_properties_per_source)
|
||||
== 0
|
||||
|
|
@ -644,11 +673,57 @@ def _scrape_onthemarket(
|
|||
except Exception as exc:
|
||||
_record_error(errors, "onthemarket", outcode, exc)
|
||||
|
||||
time.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
shutdown.sleep(DELAY_BETWEEN_OUTCODES)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _seed_detail_caches(sources: list[str], cache_dir: Path) -> None:
|
||||
"""Load each selected source's persisted detail cache into memory."""
|
||||
for source in sources:
|
||||
module = _CACHE_MODULES.get(source)
|
||||
if module is None:
|
||||
continue
|
||||
module.seed_detail_cache(load_cache(cache_dir / f"{source}.json"))
|
||||
|
||||
|
||||
def _save_detail_caches(sources: list[str], cache_dir: Path) -> None:
|
||||
"""Persist each selected source's in-memory detail cache to disk."""
|
||||
for source in sources:
|
||||
module = _CACHE_MODULES.get(source)
|
||||
if module is None:
|
||||
continue
|
||||
save_cache(cache_dir / f"{source}.json", module.detail_cache_snapshot())
|
||||
|
||||
|
||||
def _run_sources(
|
||||
run_zoopla: Callable[[], None] | None,
|
||||
background_runners: list[tuple[str, Callable[[], None]]],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
"""Run the HTTP-based providers concurrently while Zoopla runs inline.
|
||||
|
||||
Rightmove and OnTheMarket are plain-httpx and thread-safe, so each runs in a
|
||||
worker thread. Zoopla must stay on the MAIN thread because its per-outcode
|
||||
wall-clock guard uses SIGALRM, which Python only delivers there. Each source
|
||||
writes only its own ``results[source]`` list and appends to the shared
|
||||
``errors`` list (atomic under the GIL), so there is no cross-source data
|
||||
race. One source raising never kills the others: each failure is recorded
|
||||
and the remaining sources still finish."""
|
||||
with ThreadPoolExecutor(max_workers=max(1, len(background_runners))) as pool:
|
||||
futures = {pool.submit(fn): name for name, fn in background_runners}
|
||||
if run_zoopla is not None:
|
||||
try:
|
||||
run_zoopla()
|
||||
except Exception as exc: # noqa: BLE001 - one source must not kill the run
|
||||
_record_error(errors, "zoopla", "*", exc)
|
||||
for future, name in futures.items():
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc: # noqa: BLE001 - one source must not kill the run
|
||||
_record_error(errors, name, "*", exc)
|
||||
|
||||
|
||||
def run_scrape(
|
||||
outcodes: list[str],
|
||||
pc_index: PostcodeSpatialIndex,
|
||||
|
|
@ -679,28 +754,46 @@ def run_scrape(
|
|||
max_properties_per_source,
|
||||
)
|
||||
|
||||
# Persistent detail caches: load before, dump after. A listing's recovered
|
||||
# postcode never changes, so a recurring scrape only fetches new listings.
|
||||
cache_dir = output_base / "detail_cache"
|
||||
_seed_detail_caches(selected_sources, cache_dir)
|
||||
|
||||
# Rightmove and OnTheMarket are plain-httpx and run in worker threads;
|
||||
# Zoopla runs on the main thread (its SIGALRM timeout requires it). They
|
||||
# share one RATE_LIMITER, so running them at once stays polite.
|
||||
background_runners: list[tuple[str, Callable[[], None]]] = []
|
||||
if "rightmove" in selected_sources:
|
||||
_scrape_rightmove(
|
||||
selected_outcodes,
|
||||
pc_index,
|
||||
results,
|
||||
errors,
|
||||
max_properties_per_source,
|
||||
)
|
||||
|
||||
background_runners.append((
|
||||
"rightmove",
|
||||
partial(
|
||||
_scrape_rightmove,
|
||||
selected_outcodes,
|
||||
pc_index,
|
||||
results,
|
||||
errors,
|
||||
max_properties_per_source,
|
||||
),
|
||||
))
|
||||
if "onthemarket" in selected_sources:
|
||||
_scrape_onthemarket(
|
||||
selected_outcodes,
|
||||
pc_index,
|
||||
results,
|
||||
errors,
|
||||
max_properties_per_source,
|
||||
)
|
||||
background_runners.append((
|
||||
"onthemarket",
|
||||
partial(
|
||||
_scrape_onthemarket,
|
||||
selected_outcodes,
|
||||
pc_index,
|
||||
results,
|
||||
errors,
|
||||
max_properties_per_source,
|
||||
),
|
||||
))
|
||||
|
||||
run_zoopla: Callable[[], None] | None = None
|
||||
if "zoopla" in selected_sources:
|
||||
if pc_coords is None:
|
||||
pc_coords = build_postcode_coords()
|
||||
_scrape_zoopla(
|
||||
run_zoopla = partial(
|
||||
_scrape_zoopla,
|
||||
selected_outcodes,
|
||||
pc_index,
|
||||
pc_coords,
|
||||
|
|
@ -709,6 +802,11 @@ def run_scrape(
|
|||
max_properties_per_source,
|
||||
)
|
||||
|
||||
try:
|
||||
_run_sources(run_zoopla, background_runners, errors)
|
||||
finally:
|
||||
_save_detail_caches(selected_sources, cache_dir)
|
||||
|
||||
merged, source_counts, deduped = _merge_properties(results)
|
||||
output_path = output_base / "online_listings_buy.parquet"
|
||||
if merged:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue