import csv import io import zipfile from datetime import date from pathlib import Path import polars as pl from pipeline.transform.join_epc_pp import ( EPC_SOURCE_COLUMNS, _run, _scan_epc_certificates, ) def _write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, str]]) -> None: with path.open("w", newline="") as file: writer = csv.DictWriter(file, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) def _row(**overrides: str) -> dict[str, str]: row = { "address": "1 Example Street", "postcode": " aa1 1aa ", "uprn": "100012345678", "current_energy_rating": "c", "potential_energy_rating": "b", "property_type": "House", "built_form": "Mid-Terrace", "inspection_date": "2024-01-02", "total_floor_area": "84.5", "number_habitable_rooms": "5", "floor_height": "2.4", "construction_age_band": "England and Wales: 1950-1966", "tenure": "owner-occupied", } row.update(overrides) return row def test_scan_epc_certificates_supports_legacy_uppercase_csv(tmp_path: Path): csv_path = tmp_path / "certificates.csv" fieldnames = [column.upper() for column in EPC_SOURCE_COLUMNS] row = {column.upper(): value for column, value in _row().items()} row["NUMBER_HABITABLE_ROOMS"] = "0" _write_csv(csv_path, fieldnames, [row]) df = _scan_epc_certificates(csv_path, tmp_path).collect() assert df.to_dicts() == [ { "epc_address": "1 Example Street", "epc_postcode": "AA1 1AA", "uprn": "100012345678", "current_energy_rating": "C", "potential_energy_rating": "B", "epc_property_type": "House", "built_form": "Mid-Terrace", "inspection_date": date(2024, 1, 2), "total_floor_area": 84.5, "number_habitable_rooms": None, "floor_height": 2.4, "construction_age_band": "England and Wales: 1950-1966", "tenure": "owner-occupied", } ] def test_scan_epc_certificates_supports_domestic_zip(tmp_path: Path): zip_path = tmp_path / "domestic-csv.zip" rows_2023 = [_row(address="2 Example Street", inspection_date="2023-03-04")] rows_2024 = [ _row( address="3 Example Street", postcode="BB2 2BB", inspection_date="2024-05-06", total_floor_area="", tenure="Rented (social)", ) ] with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: for member_name, rows in [ ("certificates-2023.csv", rows_2023), ("nested/certificates-2024.csv", rows_2024), ]: csv_text = [",".join(EPC_SOURCE_COLUMNS)] csv_text.extend( ",".join(row[column] for column in EPC_SOURCE_COLUMNS) for row in rows ) archive.writestr(member_name, "\n".join(csv_text) + "\n") archive.writestr("recommendations-2024.csv", "address,postcode\nignored,X\n") df = _scan_epc_certificates(zip_path, tmp_path).sort("inspection_date").collect() assert df.select("epc_address", "epc_postcode", "total_floor_area").to_dicts() == [ { "epc_address": "2 Example Street", "epc_postcode": "AA1 1AA", "total_floor_area": 84.5, }, { "epc_address": "3 Example Street", "epc_postcode": "BB2 2BB", "total_floor_area": None, }, ] assert df.get_column("tenure").to_list() == ["owner-occupied", "Rented (social)"] assert df.schema["number_habitable_rooms"] == pl.Int16 def test_run_joins_domestic_zip_with_price_paid(tmp_path: Path): zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerows( [ _row( current_energy_rating="d", inspection_date="2023-01-01", total_floor_area="80", tenure="Rented (social)", ), _row( current_energy_rating="c", inspection_date="2024-01-01", total_floor_area="85", tenure="owner-occupied", ), ] ) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [200_000, 250_000], "date_of_transfer": [date(2020, 2, 3), date(2024, 2, 3)], "property_type": ["T", "T"], "postcode": ["AA1 1AA", "AA1 1AA"], "paon": ["1", "1"], "saon": [None, None], "street": ["Example-Street", "Example Street"], "locality": [None, None], "town_city": ["Exampletown", "Exampletown"], "duration": ["F", "F"], "old_new": ["N", "N"], "ppd_category": ["A", "A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 assert df.select( "epc_address", "current_energy_rating", "total_floor_area", "construction_age_band", "was_council_house", ).to_dicts() == [ { "epc_address": "1 Example Street", "current_energy_rating": "C", "total_floor_area": 85.0, # Band midpoint of 1950-1966, not the lower bound. "construction_age_band": 1958, "was_council_house": "Yes", } ] assert df.get_column("renovation_history").list.len().to_list() == [1] assert df.get_column("historical_prices").list.len().to_list() == [2] def test_run_dedup_prefers_valid_dated_cert_over_garbled_date(tmp_path: Path): # Two certificates for the same property. The cert with the garbled, # unparseable inspection_date must NOT be chosen as "latest": a string sort # nulls-first would have picked it, attaching a stale rating/floor area. The # valid-dated cert wins, so its rating ("C") and floor area (85) survive. zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerows( [ _row( current_energy_rating="c", inspection_date="2024-01-01", total_floor_area="85", ), # Same property; an unparseable date (OCR/garbled). Under a raw # string descending sort "not-a-date" outranks the ISO date and # wins the dedup, but as a null Date it loses. _row( current_energy_rating="g", inspection_date="not-a-date", total_floor_area="40", ), ] ) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [250_000], "date_of_transfer": [date(2024, 2, 3)], "property_type": ["T"], "postcode": ["AA1 1AA"], "paon": ["1"], "saon": [None], "street": ["Example Street"], "locality": [None], "town_city": ["Exampletown"], "duration": ["F"], "old_new": ["N"], "ppd_category": ["A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 # The valid-dated cert's facts are kept; the garbled-date cert is NOT chosen. assert df.select("current_energy_rating", "total_floor_area").to_dicts() == [ {"current_energy_rating": "C", "total_floor_area": 85.0} ] def test_run_excludes_price_paid_rows_without_full_postcode(tmp_path: Path): zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerow(_row()) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [250_000, 300_000], "date_of_transfer": [date(2024, 2, 3), date(2024, 2, 4)], "property_type": ["T", "T"], "postcode": ["AA1 1AA", ""], "paon": ["1", "2"], "saon": [None, None], "street": ["Example Street", "Example Street"], "locality": [None, None], "town_city": ["Exampletown", "Exampletown"], "duration": ["F", "F"], "old_new": ["N", "N"], "ppd_category": ["A", "A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df["postcode"].to_list() == ["AA1 1AA"] def test_run_does_not_attach_epc_facts_to_low_score_address_match(tmp_path: Path): zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerow(_row(address="1 Totally Different Road")) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [250_000], "date_of_transfer": [date(2024, 2, 3)], "property_type": ["T"], "postcode": ["AA1 1AA"], "paon": ["1"], "saon": [None], "street": ["Example Street"], "locality": [None], "town_city": ["Exampletown"], "duration": ["F"], "old_new": ["N"], "ppd_category": ["A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 assert df.select( "pp_address", "epc_address", "total_floor_area", "current_energy_rating", ).to_dicts() == [ { "pp_address": "1 Example Street", "epc_address": None, "total_floor_area": None, "current_energy_rating": None, } ] def test_run_excludes_category_b_sales_from_price_aggregations(tmp_path: Path): # Category B entries (repossessions, bulk/portfolio, power-of-sale) must not # pollute latest_price / historical_prices, but the property still survives # via its standard Category A sales. zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerow(_row()) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [200_000, 250_000, 5_000_000], "date_of_transfer": [date(2020, 2, 3), date(2022, 2, 3), date(2024, 2, 3)], "property_type": ["T", "T", "T"], "postcode": ["AA1 1AA", "AA1 1AA", "AA1 1AA"], "paon": ["1", "1", "1"], "saon": [None, None, None], "street": ["Example Street", "Example Street", "Example Street"], "locality": [None, None, None], "town_city": ["Exampletown", "Exampletown", "Exampletown"], "duration": ["F", "F", "F"], "old_new": ["N", "N", "N"], # The latest (5M) sale is a Category B bulk/portfolio transfer. "ppd_category": ["A", "A", "B"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 # Only the two Category A sales survive; the 5M Category B transfer is dropped. assert df.get_column("latest_price").to_list() == [250_000] assert df.get_column("historical_prices").list.len().to_list() == [2] def test_run_new_build_keeps_early_first_transfer_when_sub_min_price(tmp_path: Path): # A new-build whose earliest sale is below MIN_PRICE must still take that early # year as its EXACT construction date, while latest_price uses only the # quality-passing (>=MIN_PRICE) sale. zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerow(_row()) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { # 5_000 is below MIN_PRICE (10_000) — a nominal/junk transfer that # must still anchor the construction year but stay out of the price # aggregations. "price": [5_000, 300_000], "date_of_transfer": [date(2015, 2, 3), date(2022, 2, 3)], "property_type": ["T", "T"], "postcode": ["AA1 1AA", "AA1 1AA"], "paon": ["1", "1"], "saon": [None, None], "street": ["Example Street", "Example Street"], "locality": [None, None], "town_city": ["Exampletown", "Exampletown"], "duration": ["F", "F"], "old_new": ["Y", "Y"], "ppd_category": ["A", "A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 # Construction year is the genuine earliest transfer (2015), flagged EXACT, # even though that sale is below MIN_PRICE. assert df.get_column("construction_age_band").to_list() == [2015] assert df.get_column("is_construction_date_approximate").to_list() == [0] # latest_price uses only the >=MIN_PRICE sale; the sub-MIN sale is excluded. assert df.get_column("latest_price").to_list() == [300_000] assert df.get_column("historical_prices").list.len().to_list() == [1] def test_run_keeps_sale_above_lowered_min_price(tmp_path: Path): # A genuine cheap sale of 30_000 sits between the OLD floor (50k) and the # NEW floor (10k): it must now be RETAINED in the price aggregations. This # pins the 50k->10k change — it fails on the pre-fix 50k floor (where 30k was # excluded, giving historical_prices length 1 / latest_price 250_000). zip_path = tmp_path / "domestic-csv.zip" with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: csv_buffer = io.StringIO() writer = csv.DictWriter(csv_buffer, fieldnames=EPC_SOURCE_COLUMNS) writer.writeheader() writer.writerow(_row()) archive.writestr("certificates-2024.csv", csv_buffer.getvalue()) price_paid_path = tmp_path / "price-paid.parquet" pl.DataFrame( { "price": [250_000, 30_000], "date_of_transfer": [date(2018, 2, 3), date(2022, 2, 3)], "property_type": ["T", "T"], "postcode": ["AA1 1AA", "AA1 1AA"], "paon": ["1", "1"], "saon": [None, None], "street": ["Example Street", "Example Street"], "locality": [None, None], "town_city": ["Exampletown", "Exampletown"], "duration": ["F", "F"], "old_new": ["N", "N"], "ppd_category": ["A", "A"], } ).write_parquet(price_paid_path) output_path = tmp_path / "epc-pp.parquet" _run(zip_path, price_paid_path, output_path, tmp_path) df = pl.read_parquet(output_path) assert df.height == 1 # Both sales now survive the 10k floor; the 30_000 (2022) is the most recent. assert df.get_column("historical_prices").list.len().to_list() == [2] assert df.get_column("latest_price").to_list() == [30_000] def test_epc_band_to_year_uses_midpoint_and_clamps(): import polars as pl from pipeline.transform.join_epc_pp import epc_band_to_year df = pl.DataFrame( { "b": [ "England and Wales: 1950-1966", # midpoint 1958 "1900-1929", # midpoint 1914 "England and Wales: before 1900", # too wide -> null "2012 onwards", # single year "1012", # implausible -> null "2202", # implausible -> null None, # null -> null "1958", # already-numeric-as-string -> pass through ] } ) years = df.select(epc_band_to_year(pl.col("b")).alias("y"))["y"].to_list() assert years == [1958, 1914, None, 2012, None, None, None, 1958]