andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Format python

Browse source
This commit is contained in:
Andras Schmelczer 2026-01-31 13:07:09 +00:00
parent 85f5770e09
commit 4c258018c3
17 changed files with 348 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

100
pipeline/transform/merge.py
View file

@ -24,7 +24,9 @@ def _build_wide(
"lsoa21",
)
wide = wide.join(arcgis, on="postcode", how="inner")
print(f" {wide.shape[0]:,} rows after GPS join, {wide.estimated_size('mb'):.1f} MB")
print(
f" {wide.shape[0]:,} rows after GPS join, {wide.estimated_size('mb'):.1f} MB"
)
# Journey times (optional)
if journey_times_path and journey_times_path.exists():
@ -42,9 +44,7 @@ def _build_wide(
if iod_path and iod_path.exists():
print("Joining IoD scores...")
iod = pl.read_parquet(iod_path)
wide = wide.join(
iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left"
)
wide = wide.join(iod, left_on="lsoa21", right_on="LSOA code (2021)", how="left")
print(f" {wide.estimated_size('mb'):.1f} MB after IoD")
# POI proximity counts (pre-computed per postcode)
@ -66,44 +66,68 @@ def _build_wide(
)
# Derived columns
wide = wide.with_columns(
(pl.col("latest_price") / pl.col("total_floor_area")).alias("Price per sqm"),
).drop(
'date_of_transfer',
'inspection_date',
'floor_height',
'lsoa21',
'LSOA code (2021)',
'Local Authority District code (2024)',
'Local Authority District name (2024)',
'imd_score',
'housing_barriers_score',
'idaci_score',
'idaopi_score',
'children_young_people_score',
'adult_skills_score',
'geographical_barriers_score',
'wider_barriers_score',
).rename({
'construction_age_band': "Approximate construction age",
"income_score": "Income Score (rate)",
"employment_score": "Employment Score (rate)",
"education_score": "Education, Skills and Training Score",
"health_score": "Health Deprivation and Disability Score",
"crime_score": "Crime Score",
})
wide = (
wide.with_columns(
(pl.col("latest_price") / pl.col("total_floor_area")).alias(
"Price per sqm"
),
)
.drop(
"date_of_transfer",
"inspection_date",
"floor_height",
"lsoa21",
"LSOA code (2021)",
"Local Authority District code (2024)",
"Local Authority District name (2024)",
"imd_score",
"housing_barriers_score",
"idaci_score",
"idaopi_score",
"children_young_people_score",
"adult_skills_score",
"geographical_barriers_score",
"wider_barriers_score",
)
.rename(
{
"construction_age_band": "Approximate construction age",
"income_score": "Income Score (rate)",
"employment_score": "Employment Score (rate)",
"education_score": "Education, Skills and Training Score",
"health_score": "Health Deprivation and Disability Score",
"crime_score": "Crime Score",
}
)
)
return wide
def main():
parser = argparse.ArgumentParser(description="Build wide property dataframe with all joins")
parser.add_argument("--epc-pp", type=Path, required=True, help="EPC-Price Paid joined parquet file")
parser.add_argument("--arcgis", type=Path, required=True, help="ArcGIS postcode data parquet file")
parser.add_argument("--iod", type=Path, help="Index of Deprivation parquet file (optional)")
parser.add_argument("--poi-proximity", type=Path, help="POI proximity counts parquet file (optional)")
parser.add_argument("--journey-times", type=Path, help="Journey times parquet file (optional)")
parser.add_argument("--output", type=Path, required=True, help="Output parquet file path")
parser = argparse.ArgumentParser(
description="Build wide property dataframe with all joins"
)
parser.add_argument(
"--epc-pp", type=Path, required=True, help="EPC-Price Paid joined parquet file"
)
parser.add_argument(
"--arcgis", type=Path, required=True, help="ArcGIS postcode data parquet file"
)
parser.add_argument(
"--iod", type=Path, help="Index of Deprivation parquet file (optional)"
)
parser.add_argument(
"--poi-proximity",
type=Path,
help="POI proximity counts parquet file (optional)",
)
parser.add_argument(
"--journey-times", type=Path, help="Journey times parquet file (optional)"
)
parser.add_argument(
"--output", type=Path, required=True, help="Output parquet file path"
)
args = parser.parse_args()
wide = _build_wide(
@ -119,7 +143,7 @@ def main():
wide.write_parquet(args.output)
size_mb = args.output.stat().st_size / (1024 * 1024)
print(f"Wrote {args.output} ({size_mb:.1f} MB)")