new demo mode & tenure

pipeline/utils/fuzzy_join.py

@@ -81,6 +81,7 @@ def fuzzy_join_on_postcode(
     min_score_without_numbers: int = MIN_FUZZY_SCORE_WITHOUT_NUMBERS,
     left_variant_cols: Sequence[str] = (),
     right_variant_cols: Sequence[str] = (),
+    keep_unmatched_right: bool = False,
 ) -> pl.LazyFrame:
     """Fuzzy join two LazyFrames by matching addresses within postcode buckets.
 
@@ -106,6 +107,14 @@ def fuzzy_join_on_postcode(
     ``_match_score`` (UInt8) audit column holding the token_sort_ratio of
     the accepted match (exact matches score 100).  Unmatched rows have null
     right columns and a null score.
+
+    When ``keep_unmatched_right`` is set the join becomes a full outer join:
+    right rows that matched no left row are appended with all left columns (and
+    ``_match_score``) null. This turns "price-paid LEFT JOIN epc" into a union
+    that also surfaces EPC-only dwellings (an energy certificate but no Land
+    Registry sale). It assumes ``right`` is already unique on its match key
+    (address + postcode) — as the deduped EPC frame is — so each unmatched right
+    row appears once.
     """
 
     tmpdir = tempfile.mkdtemp(prefix="fuzzy_join_", dir=local_tmp_dir())
@@ -153,7 +162,9 @@ def fuzzy_join_on_postcode(
             .collect(engine="streaming")
         )
 
-        left_variant_names = [f"_left_variant_{i}" for i in range(len(left_variant_cols))]
+        left_variant_names = [
+            f"_left_variant_{i}" for i in range(len(left_variant_cols))
+        ]
         right_variant_names = [
             f"_right_variant_{i}" for i in range(len(right_variant_cols))
         ]
@@ -263,6 +274,21 @@ def fuzzy_join_on_postcode(
             .drop("_left_idx", "_right_idx")
             .collect(engine="streaming")
         )
+
+        if keep_unmatched_right:
+            # Right rows that matched no left row. Anti-join (not is_in) so this
+            # scales to millions of matched ids without a giant in-memory list.
+            matched_ids = pl.LazyFrame(
+                {"_right_idx": pl.Series([m[1] for m in matches], dtype=pl.UInt32)}
+            )
+            unmatched_right = (
+                right_cached.join(matched_ids, on="_right_idx", how="anti")
+                .drop("_right_idx")
+                .collect(engine="streaming")
+            )
+            # Diagonal concat fills the absent left columns (and _match_score)
+            # with null for these EPC-only rows.
+            result = pl.concat([result, unmatched_right], how="diagonal")
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)