527 lines
20 KiB
Python
527 lines
20 KiB
Python
import logging
|
|
import random
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
|
|
import polars as pl
|
|
|
|
import httpx
|
|
|
|
from constants import (
|
|
ARCGIS_PATH,
|
|
CHANNELS,
|
|
DATA_DIR,
|
|
DELAY_BETWEEN_OUTCODES,
|
|
RELOAD_URL,
|
|
SCRAPE_HOMECOUK,
|
|
SCRAPE_OPENRENT,
|
|
SCRAPE_RIGHTMOVE,
|
|
SEED,
|
|
)
|
|
from homecouk import CookiesExpiredError
|
|
from homecouk import load_cookies as load_homecouk_cookies
|
|
from homecouk import make_client as make_homecouk_client
|
|
from homecouk import search_outcode as homecouk_search_outcode
|
|
from http_client import make_client
|
|
from metrics import (
|
|
cookie_refreshes_total,
|
|
cross_source_dedup_total,
|
|
homecouk_enabled,
|
|
openrent_enabled,
|
|
scrape_elapsed_seconds,
|
|
scrape_errors_total,
|
|
scrape_outcodes_done,
|
|
scrape_outcodes_total,
|
|
scrape_properties_total,
|
|
scrape_state,
|
|
)
|
|
from openrent import WafChallengeError
|
|
from openrent import load_cookies as load_openrent_cookies
|
|
from openrent import make_client as make_openrent_client
|
|
from openrent import search_outcode as openrent_search_outcode
|
|
from rightmove import resolve_outcode_id, search_outcode
|
|
from spatial import PostcodeSpatialIndex
|
|
from storage import write_parquet
|
|
|
|
log = logging.getLogger("rightmove")
|
|
|
|
|
|
@dataclass
|
|
class ScrapeStatus:
|
|
state: str = "idle" # idle | running | done | error
|
|
channel: str = ""
|
|
outcode: str = ""
|
|
outcodes_done: int = 0
|
|
outcodes_total: int = 0
|
|
properties_buy: int = 0
|
|
properties_rent: int = 0
|
|
# Per-source counts for current channel
|
|
rm_properties: int = 0
|
|
hk_properties: int = 0
|
|
or_properties: int = 0
|
|
errors: list[str] = field(default_factory=list)
|
|
started_at: float = 0.0
|
|
finished_at: float = 0.0
|
|
|
|
|
|
status = ScrapeStatus()
|
|
status_lock = threading.Lock()
|
|
|
|
|
|
def _sync_gauges() -> None:
|
|
"""Push current ScrapeStatus values into Prometheus gauges. Must hold status_lock."""
|
|
for state in ("idle", "running", "done", "error"):
|
|
scrape_state.labels(state=state).set(1 if status.state == state else 0)
|
|
scrape_outcodes_done.set(status.outcodes_done)
|
|
scrape_outcodes_total.set(status.outcodes_total)
|
|
# Total properties (both sources combined)
|
|
scrape_properties_total.labels(channel="buy", source="total").set(
|
|
status.properties_buy
|
|
)
|
|
scrape_properties_total.labels(channel="rent", source="total").set(
|
|
status.properties_rent
|
|
)
|
|
# Per-source breakdown for current channel
|
|
ch = "buy" if status.channel == "BUY" else "rent"
|
|
scrape_properties_total.labels(channel=ch, source="rightmove").set(
|
|
status.rm_properties
|
|
)
|
|
scrape_properties_total.labels(channel=ch, source="homecouk").set(
|
|
status.hk_properties
|
|
)
|
|
scrape_properties_total.labels(channel=ch, source="openrent").set(
|
|
status.or_properties
|
|
)
|
|
if status.started_at:
|
|
end = status.finished_at if status.finished_at else time.time()
|
|
scrape_elapsed_seconds.set(end - status.started_at)
|
|
else:
|
|
scrape_elapsed_seconds.set(0)
|
|
|
|
|
|
def load_outcodes() -> list[str]:
|
|
"""Load England-only outcodes from arcgis parquet."""
|
|
log.info("Loading outcodes from %s", ARCGIS_PATH)
|
|
df = pl.read_parquet(ARCGIS_PATH, columns=["pcd", "ctry", "lat", "long"])
|
|
england = df.filter(pl.col("ctry") == "E92000001")
|
|
log.info("England postcodes: %d", len(england))
|
|
|
|
outcodes = (
|
|
england.select(
|
|
pl.col("pcd").str.extract(r"^([A-Z]{1,2}\d[A-Z0-9]?)", 1).alias("outcode")
|
|
)
|
|
.drop_nulls()
|
|
.get_column("outcode")
|
|
.unique()
|
|
.sort()
|
|
.to_list()
|
|
)
|
|
log.info("Unique England outcodes: %d", len(outcodes))
|
|
return outcodes
|
|
|
|
|
|
def build_postcode_index() -> PostcodeSpatialIndex:
|
|
"""Build spatial index from arcgis England postcodes."""
|
|
log.info("Building postcode spatial index from %s", ARCGIS_PATH)
|
|
df = pl.read_parquet(ARCGIS_PATH, columns=["pcd", "ctry", "lat", "long"])
|
|
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(
|
|
subset=["lat", "long"]
|
|
)
|
|
return PostcodeSpatialIndex(
|
|
england.get_column("lat").to_list(),
|
|
england.get_column("long").to_list(),
|
|
england.get_column("pcd").to_list(),
|
|
)
|
|
|
|
|
|
def build_postcode_coords() -> dict[str, tuple[float, float]]:
|
|
"""Build postcode → (lat, lng) lookup from arcgis England postcodes.
|
|
Used by OpenRent scraper to resolve coordinates from postcodes."""
|
|
log.info("Building postcode coords lookup from %s", ARCGIS_PATH)
|
|
df = pl.read_parquet(ARCGIS_PATH, columns=["pcd", "ctry", "lat", "long"])
|
|
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(
|
|
subset=["lat", "long"]
|
|
)
|
|
coords: dict[str, tuple[float, float]] = {}
|
|
for pcd, lat, lng in zip(
|
|
england.get_column("pcd").to_list(),
|
|
england.get_column("lat").to_list(),
|
|
england.get_column("long").to_list(),
|
|
):
|
|
coords[pcd] = (lat, lng)
|
|
log.info("Postcode coords lookup: %d postcodes", len(coords))
|
|
return coords
|
|
|
|
|
|
def _fmt_elapsed(seconds: float) -> str:
|
|
"""Format seconds as e.g. '2h13m' or '5m32s'."""
|
|
h, rem = divmod(int(seconds), 3600)
|
|
m, s = divmod(rem, 60)
|
|
if h:
|
|
return f"{h}h{m:02d}m"
|
|
return f"{m}m{s:02d}s"
|
|
|
|
|
|
def _dedup_key(p: dict) -> tuple:
|
|
"""Composite key for cross-source deduplication: (postcode, bedrooms, price).
|
|
Two listings on different portals for the same physical property will share
|
|
these attributes even though their IDs differ."""
|
|
return (p.get("Postcode", ""), p.get("Bedrooms", 0), p.get("price", 0))
|
|
|
|
|
|
def run_scrape(
|
|
outcodes: list[str],
|
|
pc_index: PostcodeSpatialIndex,
|
|
pc_coords: dict[str, tuple[float, float]] | None = None,
|
|
) -> None:
|
|
"""Main scrape loop — runs in background thread.
|
|
Scrapes Rightmove, home.co.uk, and OpenRent, merging into one dataset."""
|
|
global status
|
|
with status_lock:
|
|
status.state = "running"
|
|
status.started_at = time.time()
|
|
status.errors = []
|
|
status.properties_buy = 0
|
|
status.properties_rent = 0
|
|
_sync_gauges()
|
|
|
|
# Shuffle for geographic diversity
|
|
shuffled = list(outcodes)
|
|
random.seed(SEED)
|
|
random.shuffle(shuffled)
|
|
|
|
if not SCRAPE_RIGHTMOVE and not SCRAPE_HOMECOUK and not SCRAPE_OPENRENT:
|
|
log.warning("All scrapers disabled — nothing to do")
|
|
with status_lock:
|
|
status.state = "done"
|
|
status.finished_at = time.time()
|
|
_sync_gauges()
|
|
return
|
|
|
|
client = make_client() if SCRAPE_RIGHTMOVE else None
|
|
if not SCRAPE_RIGHTMOVE:
|
|
log.info("Rightmove scraping DISABLED (SCRAPE_RIGHTMOVE=false)")
|
|
|
|
# home.co.uk: must be enabled via SCRAPE_HOMECOUK + cookies available
|
|
hk_client = None
|
|
hk_failed = False
|
|
if not SCRAPE_HOMECOUK:
|
|
log.info("home.co.uk scraping DISABLED (SCRAPE_HOMECOUK=false)")
|
|
homecouk_enabled.set(0)
|
|
else:
|
|
hk_result = load_homecouk_cookies()
|
|
hk_client = make_homecouk_client(*hk_result) if hk_result else None
|
|
if hk_client:
|
|
log.info("home.co.uk scraping ENABLED")
|
|
homecouk_enabled.set(1)
|
|
else:
|
|
log.info(
|
|
"home.co.uk scraping DISABLED (need FlareSolverr or HOMECOUK_CF_CLEARANCE + HOMECOUK_SESSION)"
|
|
)
|
|
homecouk_enabled.set(0)
|
|
|
|
# OpenRent: must be enabled via SCRAPE_OPENRENT + cookies available
|
|
or_client = None
|
|
or_failed = False
|
|
if not SCRAPE_OPENRENT:
|
|
log.info("OpenRent scraping DISABLED (SCRAPE_OPENRENT=false)")
|
|
openrent_enabled.set(0)
|
|
else:
|
|
or_result = load_openrent_cookies()
|
|
or_client = make_openrent_client(*or_result) if or_result else None
|
|
if or_client:
|
|
log.info("OpenRent scraping ENABLED")
|
|
openrent_enabled.set(1)
|
|
else:
|
|
log.info(
|
|
"OpenRent scraping DISABLED (need FlareSolverr or OPENRENT_WAF_TOKEN)"
|
|
)
|
|
openrent_enabled.set(0)
|
|
|
|
# Build postcode coords if OpenRent is active and caller didn't provide them
|
|
if or_client and pc_coords is None:
|
|
pc_coords = build_postcode_coords()
|
|
|
|
try:
|
|
for channel_cfg in CHANNELS:
|
|
channel_name = channel_cfg["channel"]
|
|
file_suffix = "buy" if channel_name == "BUY" else "rent"
|
|
all_properties: dict[str, dict] = {} # dedup by id
|
|
seen_dedup_keys: set[tuple] = (
|
|
set()
|
|
) # cross-source dedup by (postcode, beds, price)
|
|
rm_count = 0 # Rightmove properties this channel
|
|
hk_count = 0 # home.co.uk properties this channel
|
|
hk_dedup_count = 0 # home.co.uk skipped as cross-source duplicates
|
|
or_count = 0 # OpenRent properties this channel
|
|
or_dedup_count = 0 # OpenRent skipped as cross-source duplicates
|
|
|
|
with status_lock:
|
|
status.channel = channel_name
|
|
status.outcodes_done = 0
|
|
status.outcodes_total = len(shuffled)
|
|
status.rm_properties = 0
|
|
status.hk_properties = 0
|
|
status.or_properties = 0
|
|
|
|
channel_start = time.time()
|
|
prev_prop_milestone = 0 # last 10k milestone we logged
|
|
log.info(
|
|
"=== Starting %s channel (%d outcodes) ===", channel_name, len(shuffled)
|
|
)
|
|
|
|
for i, outcode in enumerate(shuffled):
|
|
with status_lock:
|
|
status.outcode = outcode
|
|
status.outcodes_done = i
|
|
|
|
made_requests = False
|
|
|
|
# --- Rightmove ---
|
|
if SCRAPE_RIGHTMOVE:
|
|
made_requests = True
|
|
try:
|
|
outcode_id = resolve_outcode_id(client, outcode)
|
|
if not outcode_id:
|
|
log.debug(
|
|
"No Rightmove ID for outcode %s, skipping", outcode
|
|
)
|
|
else:
|
|
props = search_outcode(
|
|
client, outcode_id, outcode, channel_cfg, pc_index
|
|
)
|
|
for p in props:
|
|
pid = p["id"]
|
|
if pid not in all_properties:
|
|
all_properties[pid] = p
|
|
seen_dedup_keys.add(_dedup_key(p))
|
|
rm_count += 1
|
|
except Exception as e:
|
|
msg = f"Error scraping Rightmove {outcode}/{channel_name}: {e}"
|
|
log.error(msg)
|
|
scrape_errors_total.labels(source="rightmove").inc()
|
|
with status_lock:
|
|
status.errors.append(msg)
|
|
|
|
# --- home.co.uk ---
|
|
if hk_client and not hk_failed:
|
|
made_requests = True
|
|
try:
|
|
hk_props = homecouk_search_outcode(
|
|
hk_client,
|
|
outcode,
|
|
channel_name,
|
|
pc_index,
|
|
)
|
|
for p in hk_props:
|
|
pid = p["id"]
|
|
key = _dedup_key(p)
|
|
if pid in all_properties or key in seen_dedup_keys:
|
|
hk_dedup_count += 1
|
|
cross_source_dedup_total.labels(
|
|
channel="buy" if channel_name == "BUY" else "rent",
|
|
).inc()
|
|
continue
|
|
all_properties[pid] = p
|
|
seen_dedup_keys.add(key)
|
|
hk_count += 1
|
|
if hk_props:
|
|
log.info(
|
|
"home.co.uk %s: +%d properties", outcode, len(hk_props)
|
|
)
|
|
except CookiesExpiredError:
|
|
log.warning(
|
|
"home.co.uk cookies expired — attempting refresh via FlareSolverr"
|
|
)
|
|
hk_client.close()
|
|
hk_result = load_homecouk_cookies()
|
|
if hk_result:
|
|
hk_client = make_homecouk_client(*hk_result)
|
|
log.info("home.co.uk cookies refreshed, continuing")
|
|
cookie_refreshes_total.labels(result="success").inc()
|
|
else:
|
|
log.warning(
|
|
"Cookie refresh failed, disabling home.co.uk for rest of scrape"
|
|
)
|
|
hk_client = None
|
|
hk_failed = True
|
|
homecouk_enabled.set(0)
|
|
cookie_refreshes_total.labels(result="failure").inc()
|
|
with status_lock:
|
|
status.errors.append(
|
|
"home.co.uk cookies expired and refresh failed"
|
|
)
|
|
except Exception as e:
|
|
msg = f"Error scraping home.co.uk {outcode}/{channel_name}: {e}"
|
|
log.error(msg)
|
|
scrape_errors_total.labels(source="homecouk").inc()
|
|
with status_lock:
|
|
status.errors.append(msg)
|
|
|
|
# --- OpenRent (RENT channel only) ---
|
|
if or_client and not or_failed and channel_name == "RENT":
|
|
made_requests = True
|
|
try:
|
|
or_props = openrent_search_outcode(
|
|
or_client,
|
|
outcode,
|
|
pc_index,
|
|
pc_coords,
|
|
)
|
|
for p in or_props:
|
|
pid = p["id"]
|
|
key = _dedup_key(p)
|
|
if pid in all_properties or key in seen_dedup_keys:
|
|
or_dedup_count += 1
|
|
cross_source_dedup_total.labels(channel="rent").inc()
|
|
continue
|
|
all_properties[pid] = p
|
|
seen_dedup_keys.add(key)
|
|
or_count += 1
|
|
if or_props:
|
|
log.info(
|
|
"OpenRent %s: +%d properties", outcode, len(or_props)
|
|
)
|
|
except WafChallengeError:
|
|
log.warning(
|
|
"OpenRent WAF cookies expired — attempting refresh via FlareSolverr"
|
|
)
|
|
or_client.close()
|
|
or_result = load_openrent_cookies()
|
|
if or_result:
|
|
or_client = make_openrent_client(*or_result)
|
|
log.info("OpenRent cookies refreshed, continuing")
|
|
cookie_refreshes_total.labels(result="success").inc()
|
|
else:
|
|
log.warning(
|
|
"Cookie refresh failed, disabling OpenRent for rest of scrape"
|
|
)
|
|
or_client = None
|
|
or_failed = True
|
|
openrent_enabled.set(0)
|
|
cookie_refreshes_total.labels(result="failure").inc()
|
|
with status_lock:
|
|
status.errors.append(
|
|
"OpenRent WAF cookies expired and refresh failed"
|
|
)
|
|
except Exception as e:
|
|
msg = f"Error scraping OpenRent {outcode}/{channel_name}: {e}"
|
|
log.error(msg)
|
|
scrape_errors_total.labels(source="openrent").inc()
|
|
with status_lock:
|
|
status.errors.append(msg)
|
|
|
|
with status_lock:
|
|
if channel_name == "BUY":
|
|
status.properties_buy = len(all_properties)
|
|
else:
|
|
status.properties_rent = len(all_properties)
|
|
status.rm_properties = rm_count
|
|
status.hk_properties = hk_count
|
|
status.or_properties = or_count
|
|
_sync_gauges()
|
|
|
|
# Log progress every 100 outcodes
|
|
done = i + 1
|
|
elapsed = time.time() - channel_start
|
|
if done % 100 == 0 or done == len(shuffled):
|
|
pct = done * 100 // len(shuffled)
|
|
rate = done / elapsed if elapsed > 0 else 0
|
|
log.info(
|
|
"%s %d/%d (%d%%) — %d props, %s elapsed, %.1f outcodes/min",
|
|
channel_name,
|
|
done,
|
|
len(shuffled),
|
|
pct,
|
|
len(all_properties),
|
|
_fmt_elapsed(elapsed),
|
|
rate * 60,
|
|
)
|
|
|
|
# Log when crossing a 10k property milestone
|
|
current_milestone = len(all_properties) // 10_000
|
|
if current_milestone > prev_prop_milestone:
|
|
prev_prop_milestone = current_milestone
|
|
log.info(
|
|
"%s %dk properties (rm: %d, hk: %d, or: %d) at outcode %d/%d [%s]",
|
|
channel_name,
|
|
current_milestone * 10,
|
|
rm_count,
|
|
hk_count,
|
|
or_count,
|
|
done,
|
|
len(shuffled),
|
|
_fmt_elapsed(elapsed),
|
|
)
|
|
|
|
if made_requests and i < len(shuffled) - 1:
|
|
time.sleep(DELAY_BETWEEN_OUTCODES)
|
|
|
|
# Write parquet
|
|
deduped = list(all_properties.values())
|
|
output_path = DATA_DIR / f"online_listings_{file_suffix}.parquet"
|
|
write_parquet(deduped, output_path, channel=file_suffix)
|
|
|
|
with status_lock:
|
|
if channel_name == "BUY":
|
|
status.properties_buy = len(deduped)
|
|
else:
|
|
status.properties_rent = len(deduped)
|
|
status.outcodes_done = len(shuffled)
|
|
_sync_gauges()
|
|
|
|
log.info(
|
|
"=== %s channel complete: %d unique (rm: %d, hk: %d, or: %d, cross-dedup: %d) ===",
|
|
channel_name,
|
|
len(deduped),
|
|
rm_count,
|
|
hk_count,
|
|
or_count,
|
|
hk_dedup_count + or_dedup_count,
|
|
)
|
|
|
|
with status_lock:
|
|
status.state = "done"
|
|
status.finished_at = time.time()
|
|
_sync_gauges()
|
|
elapsed = status.finished_at - status.started_at
|
|
log.info(
|
|
"Scrape complete in %s — buy: %d, rent: %d",
|
|
_fmt_elapsed(elapsed),
|
|
status.properties_buy,
|
|
status.properties_rent,
|
|
)
|
|
|
|
# Trigger server data reload
|
|
if RELOAD_URL:
|
|
try:
|
|
log.info("Triggering server reload at %s", RELOAD_URL)
|
|
resp = httpx.post(RELOAD_URL, timeout=300)
|
|
if resp.is_success:
|
|
body = resp.json()
|
|
log.info(
|
|
"Server reload complete: %d rows, %d features, %dms",
|
|
body.get("rows", 0),
|
|
body.get("features", 0),
|
|
body.get("elapsed_ms", 0),
|
|
)
|
|
else:
|
|
log.warning("Server reload failed (%d): %s", resp.status_code, resp.text[:200])
|
|
except Exception as e:
|
|
log.warning("Server reload request failed: %s", e)
|
|
|
|
except Exception as e:
|
|
log.exception("Fatal scrape error")
|
|
with status_lock:
|
|
status.state = "error"
|
|
status.errors.append(f"Fatal: {e}")
|
|
status.finished_at = time.time()
|
|
_sync_gauges()
|
|
finally:
|
|
if client:
|
|
client.close()
|
|
if hk_client:
|
|
hk_client.close()
|
|
if or_client:
|
|
or_client.close()
|