import polars as pl
from .fuzzy_join import fuzzy_join_on_postcode


pl.Config.set_tbl_cols(-1)



epc = pl.scan_csv('data_sources/epc/certificates.csv').select(
    pl.col('ADDRESS').alias('epc_address'),
    'POSTCODE',
    'CURRENT_ENERGY_RATING',
    'POTENTIAL_ENERGY_RATING',
    pl.col('PROPERTY_TYPE').alias('epc_property_type'),
    'BUILT_FORM',
    'INSPECTION_DATE',
    'TOTAL_FLOOR_AREA',
    'NUMBER_HABITABLE_ROOMS',
    'FLOOR_HEIGHT',
    'CONSTRUCTION_AGE_BAND'
).sort('INSPECTION_DATE', descending=True).group_by('epc_address').first()


print("EPC dataset")
print(epc.head().collect())

# https://www.gov.uk/guidance/about-the-price-paid-data
property_type_map = {"D": "Detached", "S": "Semi-Detached", "T": "Terraced", "F": "Flats/Maisonettes", "O": "Other"}
duration_map = {"F": "Freehold", "L": "Leasehold"}

price_paid = (pl.scan_parquet('data_sources/pp-complete.parquet').select(
    "price",
    "date_of_transfer",
    pl.col('property_type').alias("pp_property_type").replace(property_type_map),
    "postcode",
    'paon',
    'saon',
    'street',
    'locality',
    'town_city',
    pl.col('duration').replace(duration_map)
).filter(pl.col('pp_property_type') != 'Other').with_columns(
        pl.concat_str(
            [pl.col('saon'), pl.col('paon'), pl.col('street')],
            separator=' ',
            ignore_nulls=True,
        ).alias('pp_address'),
    )
    .sort('date_of_transfer')
    .group_by('pp_address', 'postcode', maintain_order=True)
    .agg(
        pl.struct(
            pl.col('date_of_transfer').dt.year().alias('year'),
            'price',
        ).alias('historical_prices'),
        pl.col('pp_property_type').last(),
        pl.col('duration').last(),
        pl.col('price').last().alias('latest_price'),
        pl.col('date_of_transfer').last(),
    )   
)

print("Price paid dataset")
print(price_paid.head().collect())

price_paid_df = price_paid.collect()
epc_df = epc.collect()

joined = fuzzy_join_on_postcode(
    left=price_paid_df,
    right=epc_df,
    left_address_col='pp_address',
    right_address_col='epc_address',
    left_postcode_col='postcode',
    right_postcode_col='POSTCODE',
    score_threshold=80,
).drop('POSTCODE')

matched_count = joined.filter(pl.col('epc_address').is_not_null() & pl.col('pp_address').is_not_null()).height
print(f"Unique properties: {price_paid_df.height}")
print(f"Matched: {matched_count} ({100 * matched_count / price_paid_df.height:.1f}%)")
print(f"Unmatched: {price_paid_df.height - matched_count}")

joined = joined.rename({col: col.lower() for col in joined.columns})

print(joined.head())
joined.write_parquet('data_sources/processed/epc_pp.parquet')