andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Add more data

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-01 09:27:05 +00:00
parent 01edde3ebd
commit 646d8bdc46
1 changed files with 30 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

61
pipeline/transform/merge.py
View file

@ -11,8 +11,8 @@ def _build_wide(
journey_times_path: Path, journey_times_path: Path,
ethnicity_path: Path, ethnicity_path: Path,
crime_path: Path , crime_path: Path ,
noise_path: Path , noise_path: Path,
ofsted_path: Path, school_proximity_path: Path,
broadband_path: Path, broadband_path: Path,
) -> pl.DataFrame: ) -> pl.DataFrame:
"""Build the wide dataframe by joining epc_pp with all auxiliary data.""" """Build the wide dataframe by joining epc_pp with all auxiliary data."""
@ -62,31 +62,30 @@ def _build_wide(
poi_counts = pl.scan_parquet(poi_proximity_path) poi_counts = pl.scan_parquet(poi_proximity_path)
wide = wide.join(poi_counts, on="postcode", how="left") wide = wide.join(poi_counts, on="postcode", how="left")
noise = pl.scan_parquet(noise_path).select("postcode", "road_noise_lden_db") # noise = pl.scan_parquet(noise_path).select(
wide = wide.join(noise, on="postcode", how="left") # "postcode", "road_noise_lden_db", "rail_noise_lden_db", "airport_noise_lden_db"
# )
# wide = wide.join(noise, on="postcode", how="left")
print("Joining Ofsted school ratings...") print("Joining school proximity counts...")
ofsted = ( school_proximity = pl.scan_parquet(school_proximity_path)
pl.scan_parquet(ofsted_path) wide = wide.join(school_proximity, on="postcode", how="left")
.filter(pl.col("Overall effectiveness").is_in(["1", "2", "3", "4"]))
.select(
pl.col("Postcode").alias("ofsted_postcode"),
pl.col("Overall effectiveness").cast(pl.UInt8).alias("ofsted_rating"),
)
.group_by("ofsted_postcode")
.agg(pl.col("ofsted_rating").mean().round(1).alias("ofsted_avg_rating"))
)
wide = wide.join(ofsted, left_on="postcode", right_on="ofsted_postcode", how="left")
print("Joining broadband performance...") # Broadband: derive max available download speed tier per postcode from
# Ofcom availability percentages. Tiers: Gigabit ≥1000, UFBB ≥300,
# UFBB(100) ≥100, SFBB ≥30 Mbps.
print("Joining broadband availability...")
broadband = pl.scan_parquet(broadband_path).select( broadband = pl.scan_parquet(broadband_path).select(
"output_area", pl.col("postcode_space").alias("bb_postcode"),
pl.col("Average max download speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_download_900plus"), pl.when(pl.col("Gigabit availability (% premises)") > 0).then(1000)
pl.col("Average max download speed (Mbit/s) for lines 100<300Mbit/s").alias("broadband_max_download_100_300"), .when(pl.col("UFBB availability (% premises)") > 0).then(300)
pl.col("Average max download speed (Mbit/s) for lines 30<100Mbit/s").alias("broadband_max_download_30_100"), .when(pl.col("UFBB (100Mbit/s) availability (% premises)") > 0).then(100)
pl.col("Average max upload speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_upload_900plus"), .when(pl.col("SFBB availability (% premises)") > 0).then(30)
.otherwise(10)
.cast(pl.UInt16)
.alias("max_download_speed"),
) )
wide = wide.join(broadband, left_on="oa21", right_on="output_area", how="left") wide = wide.join(broadband, left_on="postcode", right_on="bb_postcode", how="left")
# Convert construction_age_band to numeric year # Convert construction_age_band to numeric year
wide = wide.with_columns( wide = wide.with_columns(
@ -153,12 +152,12 @@ def _build_wide(
"public_transport_2km": "Public transport within 2km", "public_transport_2km": "Public transport within 2km",
"latest_price": "Last known price", "latest_price": "Last known price",
"number_habitable_rooms": "Rooms (including bedrooms & bathrooms)", "number_habitable_rooms": "Rooms (including bedrooms & bathrooms)",
"road_noise_lden_db": "Road noise Lden (dB)", # "road_noise_lden_db": "Road noise Lden (dB)",
"ofsted_avg_rating": "Ofsted avg rating (1=Outstanding, 4=Inadequate)" # "rail_noise_lden_db": "Rail noise Lden (dB)",
"broadband_max_download_900plus": "Broadband download speed 900+ Mbps", # "airport_noise_lden_db": "Airport noise Lden (dB)",
"broadband_max_download_100_300": "Broadband download speed 100-300 Mbps", "good_primary_5km": "Good+ primary schools within 5km",
"broadband_max_download_30_100": "Broadband download speed 30-100 Mbps", "good_secondary_5km": "Good+ secondary schools within 5km",
"broadband_max_upload_900plus": "Broadband upload speed 900+ Mbps", "max_download_speed": "Max available download speed (Mbps)",
} }
) )
) )
@ -197,7 +196,7 @@ def main():
"--noise", type=Path, required=True, help="Road noise by postcode parquet file" "--noise", type=Path, required=True, help="Road noise by postcode parquet file"
) )
parser.add_argument( parser.add_argument(
"--ofsted", type=Path, required=True, help="Ofsted school inspection parquet file" "--school-proximity", type=Path, required=True, help="School proximity counts parquet file"
) )
parser.add_argument( parser.add_argument(
"--broadband", type=Path, required=True, help="Broadband performance by output area parquet file" "--broadband", type=Path, required=True, help="Broadband performance by output area parquet file"
@ -216,7 +215,7 @@ def main():
ethnicity_path=args.ethnicity, ethnicity_path=args.ethnicity,
crime_path=args.crime, crime_path=args.crime,
noise_path=args.noise, noise_path=args.noise,
ofsted_path=args.ofsted, school_proximity_path=args.school_proximity,
broadband_path=args.broadband, broadband_path=args.broadband,
) )