new demo mode & tenure

pipeline/transform/join_epc_pp.py

@@ -105,6 +105,35 @@ def epc_band_to_year(band: pl.Expr) -> pl.Expr:
     )
 
 
+# Coarse occupancy statuses derived from the raw EPC TENURE field, in the order
+# the timeline reads them. EPC lodges tenure as one of "Owner-occupied",
+# "Rented (private)", "Rented (social)" (plus blanks / "unknown" /
+# "Not defined - ..." for new dwellings). Anything unrecognised maps to null
+# (unknown) so it neither appears on the timeline nor breaks the change chain.
+TENURE_OWNER_OCCUPIED = "Owner-occupied"
+TENURE_RENTED_PRIVATE = "Rented (private)"
+TENURE_RENTED_SOCIAL = "Rented (social)"
+
+
+def tenure_status(tenure: pl.Expr) -> pl.Expr:
+    """Normalise the raw EPC TENURE field to a coarse occupancy status.
+
+    Matching is case-insensitive and order-sensitive: "social" is tested before
+    the generic "rent" so "Rented (social)" lands on the social bucket rather
+    than the private one. Null/blank/unrecognised tenures yield null.
+    """
+    lowered = tenure.str.to_lowercase()
+    return (
+        pl.when(lowered.str.contains("owner"))
+        .then(pl.lit(TENURE_OWNER_OCCUPIED))
+        .when(lowered.str.contains("social"))
+        .then(pl.lit(TENURE_RENTED_SOCIAL))
+        .when(lowered.str.contains("rent"))
+        .then(pl.lit(TENURE_RENTED_PRIVATE))
+        .otherwise(pl.lit(None, dtype=pl.String))
+    )
+
+
 EPC_SOURCE_COLUMNS = [
     "address",
     # The individual lines behind `address` (= address1+2+3): address2/3
@@ -484,6 +513,53 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     print(f"Renovation events: {events.height} properties with events")
     print(event_counts)
 
+    # Tenure-history fork: a chronological timeline of owner-occupied <-> rented
+    # transitions, derived from the per-certificate EPC TENURE field. Each
+    # certificate is one tenure observation; we keep only the change points so
+    # the property history can show *when* a home was let out vs lived in.
+    #
+    # Emission rule (walking certificates oldest-first, ignoring unknown-tenure
+    # ones so they neither appear nor break the chain):
+    #   - the first known status is emitted only when it is a rental — an
+    #     owner-occupied baseline is the unremarkable default for a property
+    #     that has changed hands and would only clutter the timeline;
+    #   - every later certificate is emitted when its status differs from the
+    #     previous known one (this is what surfaces the return to owner-occupied
+    #     that closes a rental period).
+    # Eager like the events/social forks: shift().over() does not run under the
+    # streaming sink used by fuzzy_join_on_postcode.
+    tenure_events = (
+        epc_base.with_columns(tenure_status(pl.col("tenure")).alias("_tenure_status"))
+        .filter(
+            pl.col("inspection_date").is_not_null()
+            & pl.col("_tenure_status").is_not_null()
+        )
+        .sort("inspection_date")
+        .with_columns(
+            pl.col("_tenure_status")
+            .shift(1)
+            .over("_epc_match_address", "_epc_match_postcode")
+            .alias("_prev_tenure_status"),
+        )
+        .filter(
+            pl.when(pl.col("_prev_tenure_status").is_null())
+            .then(pl.col("_tenure_status") != pl.lit(TENURE_OWNER_OCCUPIED))
+            .otherwise(pl.col("_tenure_status") != pl.col("_prev_tenure_status"))
+        )
+        .with_columns(
+            pl.col("inspection_date").dt.year().cast(pl.Int32).alias("_event_year"),
+        )
+        .group_by("_epc_match_address", "_epc_match_postcode")
+        .agg(
+            pl.struct(
+                pl.col("_event_year").alias("year"),
+                pl.col("_tenure_status").alias("status"),
+            ).alias("tenure_history"),
+        )
+        .collect()
+    )
+    print(f"Tenure timelines: {tenure_events.height} properties with tenure changes")
+
     # Social tenure fork: flag properties that were ever social housing
     social_tenure = (
         epc_base.filter(pl.col("tenure").str.to_lowercase().str.contains("social"))
@@ -494,13 +570,18 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     )
     print(f"Former council houses (EPC social tenure): {social_tenure.height}")
 
-    # Left-join events and social tenure back onto dedup EPC
+    # Left-join events, tenure history and social tenure back onto dedup EPC
     epc = (
         epc.join(
             events.lazy(),
             on=["_epc_match_address", "_epc_match_postcode"],
             how="left",
         )
+        .join(
+            tenure_events.lazy(),
+            on=["_epc_match_address", "_epc_match_postcode"],
+            how="left",
+        )
         .join(
             social_tenure.lazy(),
             on=["_epc_match_address", "_epc_match_postcode"],
@@ -600,9 +681,7 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     # flags are computed here and joined back into the lazy stream.
     outliers = flag_price_outliers(
         price_paid_base.filter(value_ok)
-        .select(
-            "_pp_group_address", "_pp_group_postcode", "date_of_transfer", "price"
-        )
+        .select("_pp_group_address", "_pp_group_postcode", "date_of_transfer", "price")
         .collect(engine="streaming")
     )
     print(f"Implausible consecutive-sale price jumps flagged: {outliers.height}")
@@ -665,7 +744,15 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
             right_postcode_col="epc_postcode",
             left_variant_cols=["pp_address_loc"],
             right_variant_cols=["epc_address_a1", "epc_address_a12"],
+            # Include EPC-only dwellings (an energy certificate but no Land
+            # Registry sale) so the property universe is "any dwelling we hold a
+            # record for", not just ones that have sold since 1995.
+            keep_unmatched_right=True,
         )
+        # EPC-only rows have no price-paid postcode; fall back to the EPC postcode
+        # so every row carries one (the active-English-postcode filter and the
+        # postcode->coordinates join downstream both key on it).
+        .with_columns(pl.coalesce("postcode", "epc_postcode").alias("postcode"))
         .drop("epc_postcode")
         # Audit trail: keep the fuzzy-match confidence (100 = exact address
         # match) in the published output; null means no EPC match.
@@ -676,10 +763,17 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     matched = joined.filter(
         pl.col("epc_address").is_not_null() & pl.col("pp_address").is_not_null()
     )
+    pp_only = joined.filter(
+        pl.col("pp_address").is_not_null() & pl.col("epc_address").is_null()
+    )
+    epc_only = joined.filter(pl.col("pp_address").is_null())
     total = joined.height
     print(f"Unique properties: {total}")
-    print(f"Matched: {matched.height} ({100 * matched.height / total:.1f}%)")
-    print(f"Unmatched: {total - matched.height}")
+    print(
+        f"Matched (sale + EPC): {matched.height} ({100 * matched.height / total:.1f}%)"
+    )
+    print(f"Sale only (no EPC): {pp_only.height}")
+    print(f"EPC only (never sold): {epc_only.height}")
 
     # For new-builds (old_new == "Y"), use the first transaction date year as
     # the exact construction date; otherwise fall back to the EPC age band.
@@ -689,10 +783,23 @@ def _run(epc_path: Path, price_paid_path: Path, output_path: Path, temp_dir: Pat
     )
     is_new_build = pl.col("old_new") == "Y"
 
+    # A dwelling cannot have been built after it was first sold, yet the EPC age
+    # band (a coarse range midpoint) sometimes lands later than the earliest
+    # Land Registry transfer. Cap the band-derived year at the first transfer
+    # year so the published build year is never after the first known sale. The
+    # cap only fires when both years exist: the transfer year alone is an upper
+    # bound, not an estimate, so we never fabricate a build year from it for a
+    # non-new-build that lacks an EPC band.
+    capped_band_year = (
+        pl.when(transfer_year.is_not_null() & (transfer_year < epc_band_year))
+        .then(transfer_year)
+        .otherwise(epc_band_year)
+    )
+
     joined = joined.with_columns(
         pl.when(is_new_build & transfer_year.is_not_null())
         .then(transfer_year)
-        .otherwise(epc_band_year)
+        .otherwise(capped_band_year)
         .alias("construction_age_band"),
         pl.when(is_new_build & transfer_year.is_not_null())
         .then(pl.lit(0, dtype=pl.UInt8))

pipeline/transform/merge.py

@@ -733,14 +733,14 @@ def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None
     postcode resolves into, so a missing LSOA would silently null the ethnicity
     columns for those postcodes; require full coverage instead.
     """
-    iod_lsoas = pl.read_parquet(
-        iod_path, columns=["LSOA code (2021)"]
-    ).rename({"LSOA code (2021)": "lsoa21"})
+    iod_lsoas = pl.read_parquet(iod_path, columns=["LSOA code (2021)"]).rename(
+        {"LSOA code (2021)": "lsoa21"}
+    )
 
     ethnicity_lsoas = pl.read_parquet(ethnicity_path, columns=["lsoa21"])
-    missing_ethnicity = iod_lsoas.join(
-        ethnicity_lsoas, on="lsoa21", how="anti"
-    ).sort("lsoa21")
+    missing_ethnicity = iod_lsoas.join(ethnicity_lsoas, on="lsoa21", how="anti").sort(
+        "lsoa21"
+    )
     if missing_ethnicity.height > 0:
         raise ValueError(
             "Ethnicity data is missing LSOA coverage: "
@@ -749,9 +749,7 @@ def _validate_lsoa_source_coverage(iod_path: Path, ethnicity_path: Path) -> None
         )
 
 
-def _validate_lad_source_coverage(
-    iod_path: Path, rental_prices_path: Path
-) -> None:
+def _validate_lad_source_coverage(iod_path: Path, rental_prices_path: Path) -> None:
     iod_lads = (
         pl.read_parquet(
             iod_path,
@@ -845,18 +843,32 @@ def _remap_terminated_postcodes(
 
 
 def _dedupe_collapsed_properties(wide: pl.LazyFrame) -> pl.LazyFrame:
-    """Keep one row per (postcode, pp_address) — the most-recent transaction.
+    """Keep one row per (postcode, address) — the most-recent transaction.
 
     The terminated-postcode remap can map two distinct postcodes onto one active
     successor, collapsing the same physical address onto a single
-    (postcode, pp_address) key with conflicting sale records. Keep the row with
-    the latest date_of_transfer so the headline price/date reflect the most
-    recent transaction; genuinely distinct addresses (a different pp_address) are
-    untouched. pp_address is non-null here (join_epc_pp filters it), so the key
-    never merges unrelated rows.
+    (postcode, address) key with conflicting sale records. Keep the row with the
+    latest date_of_transfer so the headline price/date reflect the most recent
+    transaction; genuinely distinct addresses are untouched.
+
+    The dedup key coalesces the price-paid address with the EPC address: EPC-only
+    dwellings (never sold) have a null pp_address, so keying on pp_address alone
+    would collapse EVERY EPC-only dwelling in a postcode onto one
+    (postcode, null) key and silently drop all but one. Each dwelling's coalesced
+    address is unique within its postcode (the EPC frame is deduped on
+    address+postcode upstream), so the coalesced key keeps them distinct while
+    leaving sold-property dedup unchanged — pp_address wins the coalesce whenever
+    a sale exists.
     """
-    return wide.sort("date_of_transfer", descending=True, nulls_last=True).unique(
-        subset=["postcode", "pp_address"], keep="first", maintain_order=True
+    return (
+        wide.with_columns(
+            pl.coalesce("pp_address", "epc_address").alias("_dedupe_address")
+        )
+        .sort("date_of_transfer", descending=True, nulls_last=True)
+        .unique(
+            subset=["postcode", "_dedupe_address"], keep="first", maintain_order=True
+        )
+        .drop("_dedupe_address")
     )
 
 

pipeline/transform/test_join_epc_pp.py

@@ -266,11 +266,79 @@ def test_run_joins_domestic_zip_with_price_paid(tmp_path: Path):
     ]
     assert df.get_column("renovation_history").list.len().to_list() == [1]
     assert df.get_column("historical_prices").list.len().to_list() == [2]
+    # Tenure timeline: the social baseline (a rental, so emitted) followed by the
+    # switch to owner-occupied that closes the rented period.
+    assert df.get_column("tenure_history").to_list() == [
+        [
+            {"year": 2023, "status": "Rented (social)"},
+            {"year": 2024, "status": "Owner-occupied"},
+        ]
+    ]
     # Audit trail: the accepted fuzzy match's score is published (100 = exact
     # post-normalisation address match).
     assert df.get_column("epc_match_score").to_list() == [100]
 
 
+def test_run_tenure_history_tracks_rent_owner_transitions(tmp_path: Path):
+    # owner-occupied (2016) -> privately rented (2019) -> owner-occupied (2023).
+    # The owner-occupied baseline is suppressed (it is the unremarkable default);
+    # the let-out and the return to owner-occupation are both surfaced so the
+    # rented period [2019, 2023) is visible on the timeline.
+    zip_path = tmp_path / "domestic-csv.zip"
+    _write_epc_zip(
+        zip_path,
+        [
+            _row(inspection_date="2016-04-01", tenure="owner-occupied"),
+            _row(inspection_date="2019-04-01", tenure="Rented (private)"),
+            _row(inspection_date="2023-04-01", tenure="owner-occupied"),
+        ],
+    )
+
+    price_paid_path = tmp_path / "price-paid.parquet"
+    _price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet(
+        price_paid_path
+    )
+
+    output_path = tmp_path / "epc-pp.parquet"
+    _run(zip_path, price_paid_path, output_path, tmp_path)
+
+    df = pl.read_parquet(output_path)
+
+    assert df.height == 1
+    assert df.get_column("tenure_history").to_list() == [
+        [
+            {"year": 2019, "status": "Rented (private)"},
+            {"year": 2023, "status": "Owner-occupied"},
+        ]
+    ]
+
+
+def test_run_tenure_history_empty_when_always_owner_occupied(tmp_path: Path):
+    # A property only ever observed as owner-occupied has no tenure change worth
+    # surfacing — the timeline column is null (no events), not a noisy baseline.
+    zip_path = tmp_path / "domestic-csv.zip"
+    _write_epc_zip(
+        zip_path,
+        [
+            _row(inspection_date="2016-04-01", tenure="owner-occupied"),
+            _row(inspection_date="2023-04-01", tenure="owner-occupied"),
+        ],
+    )
+
+    price_paid_path = tmp_path / "price-paid.parquet"
+    _price_paid_frame(prices=[250_000], dates=[date(2024, 2, 3)]).write_parquet(
+        price_paid_path
+    )
+
+    output_path = tmp_path / "epc-pp.parquet"
+    _run(zip_path, price_paid_path, output_path, tmp_path)
+
+    df = pl.read_parquet(output_path)
+
+    assert df.height == 1
+    assert df.get_column("tenure_history").to_list() == [None]
+
+
 def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path):
     # Two certificates for the same property. The cert with the garbled,
     # unparseable inspection_date must NOT be chosen as "latest": a string sort
@@ -397,8 +465,9 @@ def test_run_does_not_attach_epc_facts_to_low_score_address_match(tmp_path: Path
 
     df = pl.read_parquet(output_path)
 
-    assert df.height == 1
-    assert df.select(
+    # The low-score EPC must NOT attach its facts to the price-paid property.
+    sold = df.filter(pl.col("pp_address").is_not_null())
+    assert sold.select(
         "pp_address",
         "epc_address",
         "total_floor_area",
@@ -415,6 +484,29 @@ def test_run_does_not_attach_epc_facts_to_low_score_address_match(tmp_path: Path
         }
     ]
 
+    # Instead the unmatched EPC enters the universe as its own EPC-only row: a
+    # dwelling we hold a certificate for but that has no Land Registry sale, so
+    # its EPC facts are present while the price columns stay null.
+    epc_only = df.filter(pl.col("pp_address").is_null())
+    assert epc_only.select(
+        "epc_address",
+        "postcode",
+        "total_floor_area",
+        "current_energy_rating",
+        "latest_price",
+        "epc_match_score",
+    ).to_dicts() == [
+        {
+            "epc_address": "1 Totally Different Road",
+            "postcode": "AA1 1AA",
+            "total_floor_area": 84.5,
+            "current_energy_rating": "C",
+            "latest_price": None,
+            "epc_match_score": None,
+        }
+    ]
+    assert df.height == 2
+
 
 def test_run_excludes_category_b_sales_from_price_aggregations(tmp_path: Path):
     # Category B entries (repossessions, bulk/portfolio, power-of-sale) must not
@@ -506,6 +598,54 @@ def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: P
     assert df.get_column("historical_prices").list.len().to_list() == [1]
 
 
+def test_run_caps_band_year_at_first_transfer_year(tmp_path: Path):
+    # A non-new-build (old_new "N") whose EPC age band ("2003 onwards" -> 2003)
+    # lands AFTER its first Land Registry sale (1998). A dwelling cannot have
+    # been built after it was first sold, so the published build year must be
+    # capped at the first transfer year (1998), not the later band estimate.
+    # It stays flagged as an estimate (approximate=1) — it is still EPC-derived.
+    zip_path = tmp_path / "domestic-csv.zip"
+    _write_epc_zip(
+        zip_path, [_row(construction_age_band="England and Wales: 2003 onwards")]
+    )
+
+    price_paid_path = tmp_path / "price-paid.parquet"
+    _price_paid_frame(prices=[250_000], dates=[date(1998, 5, 1)]).write_parquet(
+        price_paid_path
+    )
+
+    output_path = tmp_path / "epc-pp.parquet"
+    _run(zip_path, price_paid_path, output_path, tmp_path)
+
+    df = pl.read_parquet(output_path)
+
+    assert df.height == 1
+    # Capped at the 1998 first sale, not the 2003 band midpoint.
+    assert df.get_column("construction_age_band").to_list() == [1998]
+    assert df.get_column("is_construction_date_approximate").to_list() == [1]
+
+
+def test_run_keeps_band_year_when_earlier_than_first_transfer(tmp_path: Path):
+    # The common case: the EPC band (1950-1966 -> 1958) predates the first
+    # recorded sale (2020). The cap must NOT fire — the band estimate stands.
+    zip_path = tmp_path / "domestic-csv.zip"
+    _write_epc_zip(zip_path)
+
+    price_paid_path = tmp_path / "price-paid.parquet"
+    _price_paid_frame(prices=[250_000], dates=[date(2020, 5, 1)]).write_parquet(
+        price_paid_path
+    )
+
+    output_path = tmp_path / "epc-pp.parquet"
+    _run(zip_path, price_paid_path, output_path, tmp_path)
+
+    df = pl.read_parquet(output_path)
+
+    assert df.height == 1
+    assert df.get_column("construction_age_band").to_list() == [1958]
+    assert df.get_column("is_construction_date_approximate").to_list() == [1]
+
+
 def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
     # A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the
     # NEW floor (10k): it must now be RETAINED in the price aggregations. This
@@ -548,13 +688,16 @@ def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path):
     assert df.get_column("latest_price").to_list() == [30_000]
 
 
-def _write_epc_zip(zip_path: Path) -> None:
-    """Write a minimal domestic zip with the default certificate row."""
+def _write_epc_zip(zip_path: Path, rows: list[dict[str, str]] | None = None) -> None:
+    """Write a minimal domestic zip with the given certificate rows.
+
+    Defaults to a single default certificate row when ``rows`` is omitted.
+    """
     with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
         csv_buffer = io.StringIO()
         writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS)
         writer.writeheader()
-        writer.writerow(_row())
+        writer.writerows(rows if rows is not None else [_row()])
         archive.writestr("certificates-2024.csv", csv_buffer.getvalue())
 
 

pipeline/transform/test_merge.py

@@ -582,9 +582,9 @@ def test_validate_lsoa_source_coverage_allows_full_ethnicity_coverage(
     )
     # Ethnicity may carry extra LSOAs (e.g. property-less ones); only the IoD
     # LSOAs are required to all be present.
-    pl.DataFrame(
-        {"lsoa21": ["E01000001", "E01000002", "E01000003"]}
-    ).write_parquet(ethnicity_path)
+    pl.DataFrame({"lsoa21": ["E01000001", "E01000002", "E01000003"]}).write_parquet(
+        ethnicity_path
+    )
 
     _validate_lsoa_source_coverage(iod_path, ethnicity_path)
 
@@ -1318,6 +1318,7 @@ def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
         {
             "postcode": ["SW3 3JY", "SW3 3JY", "SW3 3JY"],
             "pp_address": ["45 ELYSTAN PLACE", "45 ELYSTAN PLACE", "9 OTHER ROAD"],
+            "epc_address": ["45 ELYSTAN PLACE", "45 ELYSTAN PLACE", "9 OTHER ROAD"],
             "date_of_transfer": [
                 datetime(1990, 1, 1),
                 datetime(2015, 6, 1),
@@ -1339,6 +1340,39 @@ def test_dedupe_collapsed_properties_keeps_most_recent_per_address() -> None:
     assert by_addr["45 ELYSTAN PLACE"]["latest_price"] == 4_500_000
     # A genuinely distinct address in the same postcode is untouched.
     assert by_addr["9 OTHER ROAD"]["latest_price"] == 250_000
+    # The helper dedup-key column is internal and must not leak into the output.
+    assert "_dedupe_address" not in out.columns
+
+
+def test_dedupe_collapsed_properties_keeps_distinct_epc_only_dwellings() -> None:
+    # EPC-only dwellings (an energy certificate but no Land Registry sale) have a
+    # null pp_address. Keying the dedup on pp_address alone would collapse every
+    # such dwelling in a postcode onto a single (postcode, null) key and drop all
+    # but one; the coalesced (pp_address, epc_address) key keeps them distinct.
+    from datetime import datetime
+
+    wide = pl.LazyFrame(
+        {
+            "postcode": ["AB1 2CD", "AB1 2CD", "AB1 2CD"],
+            "pp_address": [None, None, "5 SOLD STREET"],
+            "epc_address": ["1 NEVER SOLD LANE", "2 NEVER SOLD LANE", None],
+            "date_of_transfer": [None, None, datetime(2010, 1, 1)],
+            "latest_price": [None, None, 300_000],
+        }
+    )
+
+    out = _dedupe_collapsed_properties(wide).collect()
+
+    # All three survive: two distinct EPC-only dwellings + one sold property.
+    assert out.height == 3
+    pairs = [
+        (row["pp_address"], row["epc_address"]) for row in out.iter_rows(named=True)
+    ]
+    assert sorted(pairs, key=lambda p: (p[0] or "", p[1] or "")) == [
+        (None, "1 NEVER SOLD LANE"),
+        (None, "2 NEVER SOLD LANE"),
+        ("5 SOLD STREET", None),
+    ]
 
 
 def _property_candidates(rows: list[dict]) -> pl.DataFrame:

pipeline/utils/fuzzy_join.py

@@ -81,6 +81,7 @@ def fuzzy_join_on_postcode(
     min_score_without_numbers: int = MIN_FUZZY_SCORE_WITHOUT_NUMBERS,
     left_variant_cols: Sequence[str] = (),
     right_variant_cols: Sequence[str] = (),
+    keep_unmatched_right: bool = False,
 ) -> pl.LazyFrame:
     """Fuzzy join two LazyFrames by matching addresses within postcode buckets.
 
@@ -106,6 +107,14 @@ def fuzzy_join_on_postcode(
     ``_match_score`` (UInt8) audit column holding the token_sort_ratio of
     the accepted match (exact matches score 100).  Unmatched rows have null
     right columns and a null score.
+
+    When ``keep_unmatched_right`` is set the join becomes a full outer join:
+    right rows that matched no left row are appended with all left columns (and
+    ``_match_score``) null. This turns "price-paid LEFT JOIN epc" into a union
+    that also surfaces EPC-only dwellings (an energy certificate but no Land
+    Registry sale). It assumes ``right`` is already unique on its match key
+    (address + postcode) — as the deduped EPC frame is — so each unmatched right
+    row appears once.
     """
 
     tmpdir = tempfile.mkdtemp(prefix="fuzzy_join_", dir=local_tmp_dir())
@@ -153,7 +162,9 @@ def fuzzy_join_on_postcode(
             .collect(engine="streaming")
         )
 
-        left_variant_names = [f"_left_variant_{i}" for i in range(len(left_variant_cols))]
+        left_variant_names = [
+            f"_left_variant_{i}" for i in range(len(left_variant_cols))
+        ]
         right_variant_names = [
             f"_right_variant_{i}" for i in range(len(right_variant_cols))
         ]
@@ -263,6 +274,21 @@ def fuzzy_join_on_postcode(
             .drop("_left_idx", "_right_idx")
             .collect(engine="streaming")
         )
+
+        if keep_unmatched_right:
+            # Right rows that matched no left row. Anti-join (not is_in) so this
+            # scales to millions of matched ids without a giant in-memory list.
+            matched_ids = pl.LazyFrame(
+                {"_right_idx": pl.Series([m[1] for m in matches], dtype=pl.UInt32)}
+            )
+            unmatched_right = (
+                right_cached.join(matched_ids, on="_right_idx", how="anti")
+                .drop("_right_idx")
+                .collect(engine="streaming")
+            )
+            # Diagonal concat fills the absent left columns (and _match_score)
+            # with null for these EPC-only rows.
+            result = pl.concat([result, unmatched_right], how="diagonal")
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)
 

pipeline/utils/test_fuzzy_join.py

@@ -220,6 +220,77 @@ def test_fuzzy_join_matches_high_score_number_less_pair():
     assert result["right_address"].to_list() == ["THE OLD RECTORY"]
 
 
+def test_fuzzy_join_keep_unmatched_right_appends_epc_only_rows():
+    # Models "price-paid LEFT JOIN epc" with keep_unmatched_right: matched pairs
+    # keep both sides; an unmatched LEFT (sold-but-no-EPC) row keeps null right
+    # columns; an unmatched RIGHT (EPC-only, never sold) row is appended with
+    # null left columns so the dwelling still enters the property universe.
+    left = pl.LazyFrame(
+        {
+            "left_id": ["sold_with_epc", "sold_no_epc"],
+            "left_address": ["10 High Street", "12 High Street"],
+            "left_postcode": ["AB1 2CD", "AB1 2CD"],
+        }
+    )
+    right = pl.LazyFrame(
+        {
+            "right_id": ["epc_for_10", "epc_only"],
+            "right_address": ["10 HIGH STREET", "14 High Street"],
+            "right_postcode": ["AB1 2CD", "AB1 2CD"],
+        }
+    )
+
+    result = (
+        fuzzy_join_on_postcode(
+            left=left,
+            right=right,
+            left_address_col="left_address",
+            right_address_col="right_address",
+            left_postcode_col="left_postcode",
+            right_postcode_col="right_postcode",
+            keep_unmatched_right=True,
+        )
+        .sort("left_id", "right_id", nulls_last=True)
+        .collect()
+    )
+
+    assert result.select("left_id", "right_id").to_dicts() == [
+        # Sold but no EPC: left present, right null (ordinary left-join row).
+        {"left_id": "sold_no_epc", "right_id": None},
+        # Matched pair: both sides present.
+        {"left_id": "sold_with_epc", "right_id": "epc_for_10"},
+        # EPC-only dwelling: appended with null left columns.
+        {"left_id": None, "right_id": "epc_only"},
+    ]
+    # The appended EPC-only row carries its own address/postcode and a null score.
+    epc_only = result.filter(pl.col("right_id") == "epc_only")
+    assert epc_only["right_address"].to_list() == ["14 High Street"]
+    assert epc_only["left_address"].to_list() == [None]
+    assert epc_only["_match_score"].to_list() == [None]
+
+
+def test_fuzzy_join_keep_unmatched_right_off_by_default():
+    # Without the flag the join stays left-only: an EPC-only right row is dropped.
+    left = pl.LazyFrame(
+        {"left_address": ["10 High Street"], "left_postcode": ["AB1 2CD"]}
+    )
+    right = pl.LazyFrame(
+        {"right_address": ["14 High Street"], "right_postcode": ["AB1 2CD"]}
+    )
+
+    result = fuzzy_join_on_postcode(
+        left=left,
+        right=right,
+        left_address_col="left_address",
+        right_address_col="right_address",
+        left_postcode_col="left_postcode",
+        right_postcode_col="right_postcode",
+    ).collect()
+
+    assert result.height == 1
+    assert result["right_address"].to_list() == [None]
+
+
 def test_numbers_compatible_treats_letter_suffix_as_part_of_the_number():
     # 8A, 8B and plain 8 are three different properties on the same street;
     # digit-only extraction collapsed all three to {8} and let them match.
@@ -252,9 +323,7 @@ def test_numbers_compatible_gates_single_letter_flats():
     assert not _numbers_compatible(
         "FLAT D 39 GERTRUDE STREET", "FLAT F 39 GERTRUDE STREET"
     )
-    assert _numbers_compatible(
-        "FLAT D 39 GERTRUDE STREET", "39 GERTRUDE STREET FLAT D"
-    )
+    assert _numbers_compatible("FLAT D 39 GERTRUDE STREET", "39 GERTRUDE STREET FLAT D")
     assert not _numbers_compatible("FLAT B ROSE COURT", "ROSE COURT")
     # A letter glued to a number ("A3") is a unit name, not a flat letter.
     assert _numbers_compatible("FLAT A3 CHESHAM HEIGHTS", "FLAT A3 CHESHAM HEIGHTS")
@@ -263,9 +332,9 @@ def test_numbers_compatible_gates_single_letter_flats():
 def test_admissible_variants_allows_locality_suffix_only():
     # Locality words may differ between a variant and its primary; digits and
     # flat designators may not (the gate ran on the primary only).
-    assert _admissible_variants(
-        "12 OAK ROAD", ["12 OAK ROAD HALE", "12 OAK ROAD"]
-    ) == ("12 OAK ROAD HALE",)
+    assert _admissible_variants("12 OAK ROAD", ["12 OAK ROAD HALE", "12 OAK ROAD"]) == (
+        "12 OAK ROAD HALE",
+    )
     # Dropping "FLAT 1" (digit) or "FLAT B" (flat designator) is inadmissible:
     # the variant would score a single flat as the whole building.
     assert (
@@ -278,9 +347,7 @@ def test_admissible_variants_allows_locality_suffix_only():
     # disagrees with the combined address must not smuggle in a different
     # street for scoring.
     assert _admissible_variants("12 OAK ROAD", ["12 ELM ROAD"]) == ()
-    assert (
-        _admissible_variants("1 TOTALLY DIFFERENT ROAD", ["1 EXAMPLE STREET"]) == ()
-    )
+    assert _admissible_variants("1 TOTALLY DIFFERENT ROAD", ["1 EXAMPLE STREET"]) == ()
 
 
 def test_fuzzy_join_variant_recovers_locality_suffix_mismatch():