fine

finder/price_history.py

@@ -0,0 +1,120 @@
+"""Forward-only asking-price history, accrued across recurring scrape runs.
+
+Rightmove (and the other portals) never expose a listing's full asking-price
+timeline: a detail page carries only the current price plus one most-recent
+"Reduced on <date>" event, and the previous price is never published. The only
+way to obtain a real "listed at X, reduced to Y" series is therefore to record
+each listing's price ourselves every run and diff it over time.
+
+This module keeps a persistent, listing-id-keyed store of price observations:
+
+    {"<listing id>": [{"date": "YYYY-MM-DD", "price": 425000, "reason": "listed"},
+                      {"date": "YYYY-MM-DD", "price": 410000, "reason": "reduced"}]}
+
+Entries are oldest -> newest. A new point is appended only when the price
+actually changes (or on first sight), so an unchanged listing costs nothing. The
+store is seeded from / dumped to disk with the same atomic JSON helpers as the
+detail-postcode caches (see postcode_cache.py), so a recurring scrape extends the
+history rather than rebuilding it.
+
+Two hard limitations, both inherent to the data source:
+  * There is NO backfill. History accrues only from the first instrumented run;
+    prior asking prices cannot be reconstructed (portals don't publish them and
+    we kept no snapshots).
+  * On a listing's first sight we know exactly one price, so we record one point.
+    If the portal's own most-recent event says that price was itself a reduction/
+    increase, we date and label that single point accordingly; we still cannot
+    invent the pre-change price.
+"""
+
+import logging
+import re
+from pathlib import Path
+
+from postcode_cache import load_cache, save_cache
+
+log = logging.getLogger("rightmove")
+
+# Portal `listingUpdateReason` codes -> our canonical reasons. Anything not a
+# recognised price move (e.g. "new", "under_offer", "auction", or absent) leaves
+# the first-sight point labelled "listed" and never fabricates a change.
+_REASON_MAP = {
+    "price_reduced": "reduced",
+    "price_increased": "increased",
+}
+
+_ISO_DATE_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})")
+
+
+def normalize_reason(raw: object) -> str | None:
+    """Map a portal update-reason code to 'reduced'/'increased', else None."""
+    if not isinstance(raw, str):
+        return None
+    return _REASON_MAP.get(raw.strip().lower())
+
+
+def _iso_to_date(value: object) -> str | None:
+    """Return the YYYY-MM-DD prefix of an ISO timestamp, or None."""
+    if not isinstance(value, str):
+        return None
+    match = _ISO_DATE_RE.match(value.strip())
+    return match.group(1) if match else None
+
+
+def load_history(path: str | Path) -> dict:
+    """Load the persisted asking-price history. Returns {} when absent/unreadable."""
+    return load_cache(path)
+
+
+def save_history(path: str | Path, history: dict) -> None:
+    """Atomically persist the asking-price history to disk."""
+    save_cache(path, history)
+
+
+def update_history(history: dict, listings: list[dict], run_date: str) -> None:
+    """Fold one run's listings into ``history`` in place.
+
+    ``run_date`` is the ISO date (YYYY-MM-DD) of the scrape. For each listing with
+    a stable id and a positive price:
+
+      * first sight -> append one point. Its date/reason come from the portal's
+        most-recent change event when that event is a price move (so a listing we
+        first meet already-reduced reads "Reduced on <event date>"); otherwise the
+        point is the "listed" price dated to ``first_visible_date`` (falling back
+        to ``run_date``).
+      * later runs -> append a point ONLY when the price differs from the last
+        recorded one, labelled "reduced"/"increased" by direction and dated to
+        ``run_date`` (we only know the change happened by this run).
+
+    An unchanged price is a no-op, so the series stays a list of genuine moves.
+    """
+    for listing in listings:
+        listing_id_raw = listing.get("id")
+        if listing_id_raw is None:
+            continue
+        listing_id = str(listing_id_raw).strip()
+        if not listing_id:
+            continue
+        try:
+            price = int(listing.get("price") or 0)
+        except (TypeError, ValueError):
+            continue
+        if price <= 0:
+            continue
+
+        entries = history.get(listing_id)
+        if not isinstance(entries, list) or not entries:
+            reason = normalize_reason(listing.get("listing_update_reason"))
+            if reason is not None:
+                date = _iso_to_date(listing.get("listing_update_date")) or run_date
+            else:
+                reason = "listed"
+                date = _iso_to_date(listing.get("first_visible_date")) or run_date
+            history[listing_id] = [{"date": date, "price": price, "reason": reason}]
+            continue
+
+        last_price = entries[-1].get("price")
+        if last_price == price:
+            continue
+        reason = "reduced" if last_price is not None and price < last_price else "increased"
+        entries.append({"date": run_date, "price": price, "reason": reason})

finder/scraper.py

@@ -5,6 +5,7 @@ import signal
 import time
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
+from datetime import datetime, timezone
 from functools import partial
 from pathlib import Path
 from typing import Callable, Iterable
@@ -34,6 +35,7 @@ from postcode_cache import load_cache, save_cache
 from rightmove import resolve_outcode_id
 from rightmove import search_outcode as rightmove_search_outcode
 from spatial import PostcodeSpatialIndex
+from price_history import load_history, save_history, update_history
 from storage import write_parquet
 from zoopla import TurnstileError
 from zoopla import launch_browser as launch_zoopla_browser
@@ -839,7 +841,18 @@ def run_scrape(
     merged, source_counts, deduped = _merge_properties(results)
     output_path = output_base / "online_listings_buy.parquet"
     if merged:
-        write_parquet(merged, output_path)
+        # Accrue the per-listing asking-price history before writing: load the
+        # persistent store, append this run's price moves, dump it, then embed
+        # each listing's series in the parquet. Forward-only by nature — the
+        # first run seeds one point per listing and reductions appear over time.
+        history_path = output_base / "price_history" / "listings.json"
+        history = load_history(history_path)
+        run_date = (
+            datetime.fromtimestamp(started_at, tz=timezone.utc).strftime("%Y-%m-%d")
+        )
+        update_history(history, merged, run_date)
+        save_history(history_path, history)
+        write_parquet(merged, output_path, price_history=history)
     else:
         if output_path.exists():
             output_path.unlink()

finder/storage.py

@@ -10,8 +10,16 @@ from transform import map_property_type, normalize_postcode
 log = logging.getLogger("rightmove")
 
 
-def write_parquet(properties: list[dict], path: Path) -> None:
-    """Write sale properties list to parquet with server-ready column names."""
+def write_parquet(
+    properties: list[dict], path: Path, price_history: dict | None = None
+) -> None:
+    """Write sale properties list to parquet with server-ready column names.
+
+    ``price_history`` is the persistent listing-id -> [{date, price, reason}]
+    store (see finder/price_history.py); each listing's accrued asking-price
+    series is embedded as a ``price_history`` list column. Absent id -> empty
+    list, so pre-instrumentation runs and tests simply write empty series.
+    """
     if not properties:
         log.warning("No properties to write to %s", path)
         return
@@ -95,6 +103,14 @@ def write_parquet(properties: list[dict], path: Path) -> None:
     asking_prices = [p["price"] if p["price"] > 0 else None for p in properties]
     listing_statuses = ["For sale"] * len(properties)
 
+    # Accrued asking-price history per listing (oldest -> newest). Look up with
+    # the SAME normalisation the store keys on (str(id).strip(), matching
+    # price_history.update_history and scraper dedup); an unseen id -> empty.
+    history_lookup = price_history or {}
+    price_history_col = [
+        history_lookup.get(str(p.get("id")).strip(), []) for p in properties
+    ]
+
     df = pl.DataFrame(
         {
             "Bedrooms": [p["Bedrooms"] for p in properties],
@@ -144,6 +160,7 @@ def write_parquet(properties: list[dict], path: Path) -> None:
             "Listing date": listing_dates,
             "Listing status": listing_statuses,
             "Asking price": asking_prices,
+            "price_history": price_history_col,
         },
         schema={
             "Bedrooms": pl.Int32,
@@ -169,6 +186,11 @@ def write_parquet(properties: list[dict], path: Path) -> None:
             "Listing date": pl.Datetime("us"),
             "Listing status": pl.Utf8,
             "Asking price": pl.Int64,
+            "price_history": pl.List(
+                pl.Struct(
+                    {"date": pl.Utf8, "price": pl.Int64, "reason": pl.Utf8}
+                )
+            ),
         },
     )
 

finder/test_price_history.py

@@ -0,0 +1,77 @@
+"""Tests for the forward-only asking-price history store (price_history.py)."""
+
+from price_history import normalize_reason, update_history
+
+
+def test_first_sight_new_listing_records_listed_at_first_visible_date() -> None:
+    history: dict = {}
+    update_history(
+        history,
+        [{"id": "A", "price": 500000, "first_visible_date": "2026-07-01T09:00:00Z"}],
+        "2026-07-12",
+    )
+    assert history["A"] == [{"date": "2026-07-01", "price": 500000, "reason": "listed"}]
+
+
+def test_first_sight_already_reduced_uses_event_date_and_reason() -> None:
+    history: dict = {}
+    update_history(
+        history,
+        [
+            {
+                "id": "B",
+                "price": 425000,
+                "first_visible_date": "2026-06-01T09:00:00Z",
+                "listing_update_reason": "price_reduced",
+                "listing_update_date": "2026-07-10T14:00:00Z",
+            }
+        ],
+        "2026-07-12",
+    )
+    assert history["B"] == [{"date": "2026-07-10", "price": 425000, "reason": "reduced"}]
+
+
+def test_later_run_appends_only_on_price_change_with_direction() -> None:
+    history: dict = {}
+    listing = {"id": "A", "price": 500000, "first_visible_date": "2026-07-01T09:00:00Z"}
+    update_history(history, [listing], "2026-07-12")
+    # Unchanged price -> no new point.
+    update_history(history, [{"id": "A", "price": 500000}], "2026-07-19")
+    assert len(history["A"]) == 1
+    # Reduced -> appended, dated to the run.
+    update_history(history, [{"id": "A", "price": 480000}], "2026-07-26")
+    # Increased -> appended.
+    update_history(history, [{"id": "A", "price": 490000}], "2026-08-02")
+    assert history["A"] == [
+        {"date": "2026-07-01", "price": 500000, "reason": "listed"},
+        {"date": "2026-07-26", "price": 480000, "reason": "reduced"},
+        {"date": "2026-08-02", "price": 490000, "reason": "increased"},
+    ]
+
+
+def test_zero_or_missing_price_and_id_are_skipped() -> None:
+    history: dict = {}
+    update_history(
+        history,
+        [
+            {"id": "POA", "price": 0},
+            {"id": "", "price": 100000},
+            {"price": 100000},  # no id
+        ],
+        "2026-07-12",
+    )
+    assert history == {}
+
+
+def test_run_date_fallback_when_dates_absent() -> None:
+    history: dict = {}
+    update_history(history, [{"id": "A", "price": 300000}], "2026-07-12")
+    assert history["A"] == [{"date": "2026-07-12", "price": 300000, "reason": "listed"}]
+
+
+def test_normalize_reason_maps_only_price_moves() -> None:
+    assert normalize_reason("price_reduced") == "reduced"
+    assert normalize_reason("price_increased") == "increased"
+    assert normalize_reason("new") is None
+    assert normalize_reason("under_offer") is None
+    assert normalize_reason(None) is None

finder/test_scraper_concurrency.py

@@ -125,7 +125,9 @@ def test_run_scrape_runs_all_sources_merges_and_persists_caches(tmp_path, monkey
     monkeypatch.setattr(
         scraper,
         "write_parquet",
-        lambda props, path: written.update(count=len(props), path=path),
+        lambda props, path, price_history=None: written.update(
+            count=len(props), path=path, price_history=price_history
+        ),
     )
 
     result = scraper.run_scrape(

finder/transform.py

@@ -385,6 +385,16 @@ def transform_property(
     if not listing_id:
         return None
 
+    # The search API already carries the single most-recent price-change event
+    # (reason + ISO date) with no extra request. Rightmove never exposes the
+    # previous price, so this only seeds the accruing asking-price history (see
+    # finder/price_history.py); it is not written to the output parquet itself.
+    listing_update = prop.get("listingUpdate")
+    if not isinstance(listing_update, dict):
+        listing_update = {}
+    listing_update_reason = listing_update.get("listingUpdateReason")
+    listing_update_date = listing_update.get("listingUpdateDate")
+
     return {
         "id": listing_id,
         "Bedrooms": bedrooms,
@@ -411,4 +421,7 @@ def transform_property(
         "Listing URL": build_listing_url(property_url, bool(prop.get("development"))),
         "Listing features": key_features,
         "first_visible_date": prop.get("firstVisibleDate", ""),
+        # Fed into the asking-price history store, not the parquet directly.
+        "listing_update_reason": listing_update_reason,
+        "listing_update_date": listing_update_date,
     }