120 lines
4.9 KiB
Python
120 lines
4.9 KiB
Python
"""Forward-only asking-price history, accrued across recurring scrape runs.
|
|
|
|
Rightmove (and the other portals) never expose a listing's full asking-price
|
|
timeline: a detail page carries only the current price plus one most-recent
|
|
"Reduced on <date>" event, and the previous price is never published. The only
|
|
way to obtain a real "listed at X, reduced to Y" series is therefore to record
|
|
each listing's price ourselves every run and diff it over time.
|
|
|
|
This module keeps a persistent, listing-id-keyed store of price observations:
|
|
|
|
{"<listing id>": [{"date": "YYYY-MM-DD", "price": 425000, "reason": "listed"},
|
|
{"date": "YYYY-MM-DD", "price": 410000, "reason": "reduced"}]}
|
|
|
|
Entries are oldest -> newest. A new point is appended only when the price
|
|
actually changes (or on first sight), so an unchanged listing costs nothing. The
|
|
store is seeded from / dumped to disk with the same atomic JSON helpers as the
|
|
detail-postcode caches (see postcode_cache.py), so a recurring scrape extends the
|
|
history rather than rebuilding it.
|
|
|
|
Two hard limitations, both inherent to the data source:
|
|
* There is NO backfill. History accrues only from the first instrumented run;
|
|
prior asking prices cannot be reconstructed (portals don't publish them and
|
|
we kept no snapshots).
|
|
* On a listing's first sight we know exactly one price, so we record one point.
|
|
If the portal's own most-recent event says that price was itself a reduction/
|
|
increase, we date and label that single point accordingly; we still cannot
|
|
invent the pre-change price.
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from postcode_cache import load_cache, save_cache
|
|
|
|
log = logging.getLogger("rightmove")
|
|
|
|
# Portal `listingUpdateReason` codes -> our canonical reasons. Anything not a
|
|
# recognised price move (e.g. "new", "under_offer", "auction", or absent) leaves
|
|
# the first-sight point labelled "listed" and never fabricates a change.
|
|
_REASON_MAP = {
|
|
"price_reduced": "reduced",
|
|
"price_increased": "increased",
|
|
}
|
|
|
|
_ISO_DATE_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})")
|
|
|
|
|
|
def normalize_reason(raw: object) -> str | None:
|
|
"""Map a portal update-reason code to 'reduced'/'increased', else None."""
|
|
if not isinstance(raw, str):
|
|
return None
|
|
return _REASON_MAP.get(raw.strip().lower())
|
|
|
|
|
|
def _iso_to_date(value: object) -> str | None:
|
|
"""Return the YYYY-MM-DD prefix of an ISO timestamp, or None."""
|
|
if not isinstance(value, str):
|
|
return None
|
|
match = _ISO_DATE_RE.match(value.strip())
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def load_history(path: str | Path) -> dict:
|
|
"""Load the persisted asking-price history. Returns {} when absent/unreadable."""
|
|
return load_cache(path)
|
|
|
|
|
|
def save_history(path: str | Path, history: dict) -> None:
|
|
"""Atomically persist the asking-price history to disk."""
|
|
save_cache(path, history)
|
|
|
|
|
|
def update_history(history: dict, listings: list[dict], run_date: str) -> None:
|
|
"""Fold one run's listings into ``history`` in place.
|
|
|
|
``run_date`` is the ISO date (YYYY-MM-DD) of the scrape. For each listing with
|
|
a stable id and a positive price:
|
|
|
|
* first sight -> append one point. Its date/reason come from the portal's
|
|
most-recent change event when that event is a price move (so a listing we
|
|
first meet already-reduced reads "Reduced on <event date>"); otherwise the
|
|
point is the "listed" price dated to ``first_visible_date`` (falling back
|
|
to ``run_date``).
|
|
* later runs -> append a point ONLY when the price differs from the last
|
|
recorded one, labelled "reduced"/"increased" by direction and dated to
|
|
``run_date`` (we only know the change happened by this run).
|
|
|
|
An unchanged price is a no-op, so the series stays a list of genuine moves.
|
|
"""
|
|
for listing in listings:
|
|
listing_id_raw = listing.get("id")
|
|
if listing_id_raw is None:
|
|
continue
|
|
listing_id = str(listing_id_raw).strip()
|
|
if not listing_id:
|
|
continue
|
|
try:
|
|
price = int(listing.get("price") or 0)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if price <= 0:
|
|
continue
|
|
|
|
entries = history.get(listing_id)
|
|
if not isinstance(entries, list) or not entries:
|
|
reason = normalize_reason(listing.get("listing_update_reason"))
|
|
if reason is not None:
|
|
date = _iso_to_date(listing.get("listing_update_date")) or run_date
|
|
else:
|
|
reason = "listed"
|
|
date = _iso_to_date(listing.get("first_visible_date")) or run_date
|
|
history[listing_id] = [{"date": date, "price": price, "reason": reason}]
|
|
continue
|
|
|
|
last_price = entries[-1].get("price")
|
|
if last_price == price:
|
|
continue
|
|
reason = "reduced" if last_price is not None and price < last_price else "increased"
|
|
entries.append({"date": run_date, "price": price, "reason": reason})
|