fine

pipeline/transform/merge.py

@@ -1018,6 +1018,14 @@ _LISTING_OVERLAY_SOURCES: tuple[tuple[str, str, pl.DataType], ...] = (
     ("Listing date", "_actual_listing_date", pl.Datetime("us")),
     ("Listing status", "_actual_listing_status", pl.Utf8),
     ("Listing features", "_actual_listing_features", pl.List(pl.Utf8)),
+    # Accrued asking-price history (oldest -> newest) from the scraper's
+    # forward-only store (finder/price_history.py). Carried through untouched and
+    # surfaced verbatim in `_finalize_listings`.
+    (
+        "price_history",
+        "_actual_price_history",
+        pl.List(pl.Struct({"date": pl.Utf8, "price": pl.Int64, "reason": pl.Utf8})),
+    ),
     ("Bedrooms", "_actual_bedrooms", pl.Int32),
     ("Bathrooms", "_actual_bathrooms", pl.Int32),
     ("Price qualifier", "_actual_price_qualifier", pl.Utf8),
@@ -1521,7 +1529,14 @@ def _load_listings_for_merge(listings_path: Path, arcgis_path: Path) -> pl.DataF
     # treats NaN as distinct from null and the downstream `latest_price /
     # total_floor_area` cast to Int32 explodes on a NaN, so we normalise floats
     # to null at load time.
+    raw_columns = set(raw.collect_schema().names())
+
     def _overlay_expr(src: str, dst: str, dtype: pl.DataType) -> pl.Expr:
+        # A listings parquet written before a column existed (e.g. price_history)
+        # must not crash the merge: substitute a typed-null overlay so the rest of
+        # the pipeline sees a well-typed, empty column instead of ColumnNotFound.
+        if src not in raw_columns:
+            return pl.lit(None, dtype=dtype).alias(dst)
         expr = pl.col(src).cast(dtype, strict=False)
         if dtype in (pl.Float32, pl.Float64):
             expr = expr.fill_nan(None)
@@ -2196,6 +2211,7 @@ def _finalize_listings(df: pl.DataFrame) -> pl.DataFrame:
         pl.col("_actual_listing_date").alias("Listing date"),
         pl.col("_actual_listing_status").alias("Listing status"),
         pl.col("_actual_listing_features").alias("Listing features"),
+        pl.col("_actual_price_history").alias("price_history"),
         pl.col("_actual_asking_price").alias("Asking price"),
         pl.col("_actual_asking_price_per_sqm").alias("Asking price per sqm"),
         pl.col("_actual_bedrooms").alias("Bedrooms"),