new demo mode & tenure

pipeline/utils/fuzzy_join.py

@@ -81,6 +81,7 @@ def fuzzy_join_on_postcode(
     min_score_without_numbers: int = MIN_FUZZY_SCORE_WITHOUT_NUMBERS,
     left_variant_cols: Sequence[str] = (),
     right_variant_cols: Sequence[str] = (),
+    keep_unmatched_right: bool = False,
 ) -> pl.LazyFrame:
     """Fuzzy join two LazyFrames by matching addresses within postcode buckets.
 
@@ -106,6 +107,14 @@ def fuzzy_join_on_postcode(
     ``_match_score`` (UInt8) audit column holding the token_sort_ratio of
     the accepted match (exact matches score 100).  Unmatched rows have null
     right columns and a null score.
+
+    When ``keep_unmatched_right`` is set the join becomes a full outer join:
+    right rows that matched no left row are appended with all left columns (and
+    ``_match_score``) null. This turns "price-paid LEFT JOIN epc" into a union
+    that also surfaces EPC-only dwellings (an energy certificate but no Land
+    Registry sale). It assumes ``right`` is already unique on its match key
+    (address + postcode) — as the deduped EPC frame is — so each unmatched right
+    row appears once.
     """
 
     tmpdir = tempfile.mkdtemp(prefix="fuzzy_join_", dir=local_tmp_dir())
@@ -153,7 +162,9 @@ def fuzzy_join_on_postcode(
             .collect(engine="streaming")
         )
 
-        left_variant_names = [f"_left_variant_{i}" for i in range(len(left_variant_cols))]
+        left_variant_names = [
+            f"_left_variant_{i}" for i in range(len(left_variant_cols))
+        ]
         right_variant_names = [
             f"_right_variant_{i}" for i in range(len(right_variant_cols))
         ]
@@ -263,6 +274,21 @@ def fuzzy_join_on_postcode(
             .drop("_left_idx", "_right_idx")
             .collect(engine="streaming")
         )
+
+        if keep_unmatched_right:
+            # Right rows that matched no left row. Anti-join (not is_in) so this
+            # scales to millions of matched ids without a giant in-memory list.
+            matched_ids = pl.LazyFrame(
+                {"_right_idx": pl.Series([m[1] for m in matches], dtype=pl.UInt32)}
+            )
+            unmatched_right = (
+                right_cached.join(matched_ids, on="_right_idx", how="anti")
+                .drop("_right_idx")
+                .collect(engine="streaming")
+            )
+            # Diagonal concat fills the absent left columns (and _match_score)
+            # with null for these EPC-only rows.
+            result = pl.concat([result, unmatched_right], how="diagonal")
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
 

pipeline/utils/test_fuzzy_join.py

@@ -220,6 +220,77 @@ def test_fuzzy_join_matches_high_score_number_less_pair():
     assert result["right_address"].to_list() == ["THE OLD RECTORY"]
 
 
+def test_fuzzy_join_keep_unmatched_right_appends_epc_only_rows():
+    # Models "price-paid LEFT JOIN epc" with keep_unmatched_right: matched pairs
+    # keep both sides; an unmatched LEFT (sold-but-no-EPC) row keeps null right
+    # columns; an unmatched RIGHT (EPC-only, never sold) row is appended with
+    # null left columns so the dwelling still enters the property universe.
+    left = pl.LazyFrame(
+        {
+            "left_id": ["sold_with_epc", "sold_no_epc"],
+            "left_address": ["10 High Street", "12 High Street"],
+            "left_postcode": ["AB1 2CD", "AB1 2CD"],
+        }
+    )
+    right = pl.LazyFrame(
+        {
+            "right_id": ["epc_for_10", "epc_only"],
+            "right_address": ["10 HIGH STREET", "14 High Street"],
+            "right_postcode": ["AB1 2CD", "AB1 2CD"],
+        }
+    )
+
+    result = (
+        fuzzy_join_on_postcode(
+            left=left,
+            right=right,
+            left_address_col="left_address",
+            right_address_col="right_address",
+            left_postcode_col="left_postcode",
+            right_postcode_col="right_postcode",
+            keep_unmatched_right=True,
+        )
+        .sort("left_id", "right_id", nulls_last=True)
+        .collect()
+    )
+
+    assert result.select("left_id", "right_id").to_dicts() == [
+        # Sold but no EPC: left present, right null (ordinary left-join row).
+        {"left_id": "sold_no_epc", "right_id": None},
+        # Matched pair: both sides present.
+        {"left_id": "sold_with_epc", "right_id": "epc_for_10"},
+        # EPC-only dwelling: appended with null left columns.
+        {"left_id": None, "right_id": "epc_only"},
+    ]
+    # The appended EPC-only row carries its own address/postcode and a null score.
+    epc_only = result.filter(pl.col("right_id") == "epc_only")
+    assert epc_only["right_address"].to_list() == ["14 High Street"]
+    assert epc_only["left_address"].to_list() == [None]
+    assert epc_only["_match_score"].to_list() == [None]
+
+
+def test_fuzzy_join_keep_unmatched_right_off_by_default():
+    # Without the flag the join stays left-only: an EPC-only right row is dropped.
+    left = pl.LazyFrame(
+        {"left_address": ["10 High Street"], "left_postcode": ["AB1 2CD"]}
+    )
+    right = pl.LazyFrame(
+        {"right_address": ["14 High Street"], "right_postcode": ["AB1 2CD"]}
+    )
+
+    result = fuzzy_join_on_postcode(
+        left=left,
+        right=right,
+        left_address_col="left_address",
+        right_address_col="right_address",
+        left_postcode_col="left_postcode",
+        right_postcode_col="right_postcode",
+    ).collect()
+
+    assert result.height == 1
+    assert result["right_address"].to_list() == [None]
+
+
 def test_numbers_compatible_treats_letter_suffix_as_part_of_the_number():
     # 8A, 8B and plain 8 are three different properties on the same street;
     # digit-only extraction collapsed all three to {8} and let them match.
@@ -252,9 +323,7 @@ def test_numbers_compatible_gates_single_letter_flats():
     assert not _numbers_compatible(
         "FLAT D 39 GERTRUDE STREET", "FLAT F 39 GERTRUDE STREET"
     )
-    assert _numbers_compatible(
-        "FLAT D 39 GERTRUDE STREET", "39 GERTRUDE STREET FLAT D"
-    )
+    assert _numbers_compatible("FLAT D 39 GERTRUDE STREET", "39 GERTRUDE STREET FLAT D")
     assert not _numbers_compatible("FLAT B ROSE COURT", "ROSE COURT")
     # A letter glued to a number ("A3") is a unit name, not a flat letter.
     assert _numbers_compatible("FLAT A3 CHESHAM HEIGHTS", "FLAT A3 CHESHAM HEIGHTS")
@@ -263,9 +332,9 @@ def test_numbers_compatible_gates_single_letter_flats():
 def test_admissible_variants_allows_locality_suffix_only():
     # Locality words may differ between a variant and its primary; digits and
     # flat designators may not (the gate ran on the primary only).
-    assert _admissible_variants(
-        "12 OAK ROAD", ["12 OAK ROAD HALE", "12 OAK ROAD"]
-    ) == ("12 OAK ROAD HALE",)
+    assert _admissible_variants("12 OAK ROAD", ["12 OAK ROAD HALE", "12 OAK ROAD"]) == (
+        "12 OAK ROAD HALE",
+    )
     # Dropping "FLAT 1" (digit) or "FLAT B" (flat designator) is inadmissible:
     # the variant would score a single flat as the whole building.
     assert (
@@ -278,9 +347,7 @@ def test_admissible_variants_allows_locality_suffix_only():
     # disagrees with the combined address must not smuggle in a different
     # street for scoring.
     assert _admissible_variants("12 OAK ROAD", ["12 ELM ROAD"]) == ()
-    assert (
-        _admissible_variants("1 TOTALLY DIFFERENT ROAD", ["1 EXAMPLE STREET"]) == ()
-    )
+    assert _admissible_variants("1 TOTALLY DIFFERENT ROAD", ["1 EXAMPLE STREET"]) == ()
 
 
 def test_fuzzy_join_variant_recovers_locality_suffix_mismatch():