fine

finder/price_history.py

@@ -0,0 +1,120 @@
+"""Forward-only asking-price history, accrued across recurring scrape runs.
+
+Rightmove (and the other portals) never expose a listing's full asking-price
+timeline: a detail page carries only the current price plus one most-recent
+"Reduced on <date>" event, and the previous price is never published. The only
+way to obtain a real "listed at X, reduced to Y" series is therefore to record
+each listing's price ourselves every run and diff it over time.
+
+This module keeps a persistent, listing-id-keyed store of price observations:
+
+    {"<listing id>": [{"date": "YYYY-MM-DD", "price": 425000, "reason": "listed"},
+                      {"date": "YYYY-MM-DD", "price": 410000, "reason": "reduced"}]}
+
+Entries are oldest -> newest. A new point is appended only when the price
+actually changes (or on first sight), so an unchanged listing costs nothing. The
+store is seeded from / dumped to disk with the same atomic JSON helpers as the
+detail-postcode caches (see postcode_cache.py), so a recurring scrape extends the
+history rather than rebuilding it.
+
+Two hard limitations, both inherent to the data source:
+  * There is NO backfill. History accrues only from the first instrumented run;
+    prior asking prices cannot be reconstructed (portals don't publish them and
+    we kept no snapshots).
+  * On a listing's first sight we know exactly one price, so we record one point.
+    If the portal's own most-recent event says that price was itself a reduction/
+    increase, we date and label that single point accordingly; we still cannot
+    invent the pre-change price.
+"""
+
+import logging
+import re
+from pathlib import Path
+
+from postcode_cache import load_cache, save_cache
+
+log = logging.getLogger("rightmove")
+
+# Portal `listingUpdateReason` codes -> our canonical reasons. Anything not a
+# recognised price move (e.g. "new", "under_offer", "auction", or absent) leaves
+# the first-sight point labelled "listed" and never fabricates a change.
+_REASON_MAP = {
+    "price_reduced": "reduced",
+    "price_increased": "increased",
+}
+
+_ISO_DATE_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})")
+
+
+def normalize_reason(raw: object) -> str | None:
+    """Map a portal update-reason code to 'reduced'/'increased', else None."""
+    if not isinstance(raw, str):
+        return None
+    return _REASON_MAP.get(raw.strip().lower())
+
+
+def _iso_to_date(value: object) -> str | None:
+    """Return the YYYY-MM-DD prefix of an ISO timestamp, or None."""
+    if not isinstance(value, str):
+        return None
+    match = _ISO_DATE_RE.match(value.strip())
+    return match.group(1) if match else None
+
+
+def load_history(path: str | Path) -> dict:
+    """Load the persisted asking-price history. Returns {} when absent/unreadable."""
+    return load_cache(path)
+
+
+def save_history(path: str | Path, history: dict) -> None:
+    """Atomically persist the asking-price history to disk."""
+    save_cache(path, history)
+
+
+def update_history(history: dict, listings: list[dict], run_date: str) -> None:
+    """Fold one run's listings into ``history`` in place.
+
+    ``run_date`` is the ISO date (YYYY-MM-DD) of the scrape. For each listing with
+    a stable id and a positive price:
+
+      * first sight -> append one point. Its date/reason come from the portal's
+        most-recent change event when that event is a price move (so a listing we
+        first meet already-reduced reads "Reduced on <event date>"); otherwise the
+        point is the "listed" price dated to ``first_visible_date`` (falling back
+        to ``run_date``).
+      * later runs -> append a point ONLY when the price differs from the last
+        recorded one, labelled "reduced"/"increased" by direction and dated to
+        ``run_date`` (we only know the change happened by this run).
+
+    An unchanged price is a no-op, so the series stays a list of genuine moves.
+    """
+    for listing in listings:
+        listing_id_raw = listing.get("id")
+        if listing_id_raw is None:
+            continue
+        listing_id = str(listing_id_raw).strip()
+        if not listing_id:
+            continue
+        try:
+            price = int(listing.get("price") or 0)
+        except (TypeError, ValueError):
+            continue
+        if price <= 0:
+            continue
+
+        entries = history.get(listing_id)
+        if not isinstance(entries, list) or not entries:
+            reason = normalize_reason(listing.get("listing_update_reason"))
+            if reason is not None:
+                date = _iso_to_date(listing.get("listing_update_date")) or run_date
+            else:
+                reason = "listed"
+                date = _iso_to_date(listing.get("first_visible_date")) or run_date
+            history[listing_id] = [{"date": date, "price": price, "reason": reason}]
+            continue
+
+        last_price = entries[-1].get("price")
+        if last_price == price:
+            continue
+        reason = "reduced" if last_price is not None and price < last_price else "increased"
+        entries.append({"date": run_date, "price": price, "reason": reason})