diff --git a/pipeline/transform/merge.py b/pipeline/transform/merge.py index e672b78..ffbb046 100644 --- a/pipeline/transform/merge.py +++ b/pipeline/transform/merge.py @@ -11,8 +11,8 @@ def _build_wide( journey_times_path: Path, ethnicity_path: Path, crime_path: Path , - noise_path: Path , - ofsted_path: Path, + noise_path: Path, + school_proximity_path: Path, broadband_path: Path, ) -> pl.DataFrame: """Build the wide dataframe by joining epc_pp with all auxiliary data.""" @@ -62,31 +62,30 @@ def _build_wide( poi_counts = pl.scan_parquet(poi_proximity_path) wide = wide.join(poi_counts, on="postcode", how="left") - noise = pl.scan_parquet(noise_path).select("postcode", "road_noise_lden_db") - wide = wide.join(noise, on="postcode", how="left") + # noise = pl.scan_parquet(noise_path).select( + # "postcode", "road_noise_lden_db", "rail_noise_lden_db", "airport_noise_lden_db" + # ) + # wide = wide.join(noise, on="postcode", how="left") - print("Joining Ofsted school ratings...") - ofsted = ( - pl.scan_parquet(ofsted_path) - .filter(pl.col("Overall effectiveness").is_in(["1", "2", "3", "4"])) - .select( - pl.col("Postcode").alias("ofsted_postcode"), - pl.col("Overall effectiveness").cast(pl.UInt8).alias("ofsted_rating"), - ) - .group_by("ofsted_postcode") - .agg(pl.col("ofsted_rating").mean().round(1).alias("ofsted_avg_rating")) - ) - wide = wide.join(ofsted, left_on="postcode", right_on="ofsted_postcode", how="left") + print("Joining school proximity counts...") + school_proximity = pl.scan_parquet(school_proximity_path) + wide = wide.join(school_proximity, on="postcode", how="left") - print("Joining broadband performance...") + # Broadband: derive max available download speed tier per postcode from + # Ofcom availability percentages. Tiers: Gigabit ≥1000, UFBB ≥300, + # UFBB(100) ≥100, SFBB ≥30 Mbps. + print("Joining broadband availability...") broadband = pl.scan_parquet(broadband_path).select( - "output_area", - pl.col("Average max download speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_download_900plus"), - pl.col("Average max download speed (Mbit/s) for lines 100<300Mbit/s").alias("broadband_max_download_100_300"), - pl.col("Average max download speed (Mbit/s) for lines 30<100Mbit/s").alias("broadband_max_download_30_100"), - pl.col("Average max upload speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_upload_900plus"), + pl.col("postcode_space").alias("bb_postcode"), + pl.when(pl.col("Gigabit availability (% premises)") > 0).then(1000) + .when(pl.col("UFBB availability (% premises)") > 0).then(300) + .when(pl.col("UFBB (100Mbit/s) availability (% premises)") > 0).then(100) + .when(pl.col("SFBB availability (% premises)") > 0).then(30) + .otherwise(10) + .cast(pl.UInt16) + .alias("max_download_speed"), ) - wide = wide.join(broadband, left_on="oa21", right_on="output_area", how="left") + wide = wide.join(broadband, left_on="postcode", right_on="bb_postcode", how="left") # Convert construction_age_band to numeric year wide = wide.with_columns( @@ -153,12 +152,12 @@ def _build_wide( "public_transport_2km": "Public transport within 2km", "latest_price": "Last known price", "number_habitable_rooms": "Rooms (including bedrooms & bathrooms)", - "road_noise_lden_db": "Road noise Lden (dB)", - "ofsted_avg_rating": "Ofsted avg rating (1=Outstanding, 4=Inadequate)" - "broadband_max_download_900plus": "Broadband download speed 900+ Mbps", - "broadband_max_download_100_300": "Broadband download speed 100-300 Mbps", - "broadband_max_download_30_100": "Broadband download speed 30-100 Mbps", - "broadband_max_upload_900plus": "Broadband upload speed 900+ Mbps", + # "road_noise_lden_db": "Road noise Lden (dB)", + # "rail_noise_lden_db": "Rail noise Lden (dB)", + # "airport_noise_lden_db": "Airport noise Lden (dB)", + "good_primary_5km": "Good+ primary schools within 5km", + "good_secondary_5km": "Good+ secondary schools within 5km", + "max_download_speed": "Max available download speed (Mbps)", } ) ) @@ -197,7 +196,7 @@ def main(): "--noise", type=Path, required=True, help="Road noise by postcode parquet file" ) parser.add_argument( - "--ofsted", type=Path, required=True, help="Ofsted school inspection parquet file" + "--school-proximity", type=Path, required=True, help="School proximity counts parquet file" ) parser.add_argument( "--broadband", type=Path, required=True, help="Broadband performance by output area parquet file" @@ -216,7 +215,7 @@ def main(): ethnicity_path=args.ethnicity, crime_path=args.crime, noise_path=args.noise, - ofsted_path=args.ofsted, + school_proximity_path=args.school_proximity, broadband_path=args.broadband, )