Add more data

pipeline/transform/merge.py

@@ -11,8 +11,8 @@ def _build_wide(
     journey_times_path: Path,
     ethnicity_path: Path,
     crime_path: Path ,
-    noise_path: Path ,
-    ofsted_path: Path,
+    noise_path: Path,
+    school_proximity_path: Path,
     broadband_path: Path,
 ) -> pl.DataFrame:
     """Build the wide dataframe by joining epc_pp with all auxiliary data."""
@@ -62,31 +62,30 @@ def _build_wide(
     poi_counts = pl.scan_parquet(poi_proximity_path)
     wide = wide.join(poi_counts, on="postcode", how="left")
 
-    noise = pl.scan_parquet(noise_path).select("postcode", "road_noise_lden_db")
-    wide = wide.join(noise, on="postcode", how="left")
+    # noise = pl.scan_parquet(noise_path).select(
+    #     "postcode", "road_noise_lden_db", "rail_noise_lden_db", "airport_noise_lden_db"
+    # )
+    # wide = wide.join(noise, on="postcode", how="left")
 
-    print("Joining Ofsted school ratings...")
-    ofsted = (
-        pl.scan_parquet(ofsted_path)
-        .filter(pl.col("Overall effectiveness").is_in(["1", "2", "3", "4"]))
-        .select(
-            pl.col("Postcode").alias("ofsted_postcode"),
-            pl.col("Overall effectiveness").cast(pl.UInt8).alias("ofsted_rating"),
-        )
-        .group_by("ofsted_postcode")
-        .agg(pl.col("ofsted_rating").mean().round(1).alias("ofsted_avg_rating"))
-    )
-    wide = wide.join(ofsted, left_on="postcode", right_on="ofsted_postcode", how="left")
+    print("Joining school proximity counts...")
+    school_proximity = pl.scan_parquet(school_proximity_path)
+    wide = wide.join(school_proximity, on="postcode", how="left")
 
-    print("Joining broadband performance...")
+    # Broadband: derive max available download speed tier per postcode from
+    # Ofcom availability percentages.  Tiers: Gigabit ≥1000, UFBB ≥300,
+    # UFBB(100) ≥100, SFBB ≥30 Mbps.
+    print("Joining broadband availability...")
     broadband = pl.scan_parquet(broadband_path).select(
-        "output_area",
-        pl.col("Average max download speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_download_900plus"),
-        pl.col("Average max download speed (Mbit/s) for lines 100<300Mbit/s").alias("broadband_max_download_100_300"),
-        pl.col("Average max download speed (Mbit/s) for lines 30<100Mbit/s").alias("broadband_max_download_30_100"),
-        pl.col("Average max upload speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_upload_900plus"),
+        pl.col("postcode_space").alias("bb_postcode"),
+        pl.when(pl.col("Gigabit availability (% premises)") > 0).then(1000)
+          .when(pl.col("UFBB availability (% premises)") > 0).then(300)
+          .when(pl.col("UFBB (100Mbit/s) availability (% premises)") > 0).then(100)
+          .when(pl.col("SFBB availability (% premises)") > 0).then(30)
+          .otherwise(10)
+          .cast(pl.UInt16)
+          .alias("max_download_speed"),
     )
-    wide = wide.join(broadband, left_on="oa21", right_on="output_area", how="left")
+    wide = wide.join(broadband, left_on="postcode", right_on="bb_postcode", how="left")
 
     # Convert construction_age_band to numeric year
     wide = wide.with_columns(
@@ -153,12 +152,12 @@ def _build_wide(
                 "public_transport_2km": "Public transport within 2km",
                 "latest_price": "Last known price",
                 "number_habitable_rooms": "Rooms (including bedrooms & bathrooms)",
-                "road_noise_lden_db": "Road noise Lden (dB)",
-                "ofsted_avg_rating": "Ofsted avg rating (1=Outstanding, 4=Inadequate)"
-                        "broadband_max_download_900plus": "Broadband download speed 900+ Mbps",
-                        "broadband_max_download_100_300": "Broadband download speed 100-300 Mbps",
-                        "broadband_max_download_30_100": "Broadband download speed 30-100 Mbps",
-                        "broadband_max_upload_900plus": "Broadband upload speed 900+ Mbps",
+                # "road_noise_lden_db": "Road noise Lden (dB)",
+                # "rail_noise_lden_db": "Rail noise Lden (dB)",
+                # "airport_noise_lden_db": "Airport noise Lden (dB)",
+                "good_primary_5km": "Good+ primary schools within 5km",
+                "good_secondary_5km": "Good+ secondary schools within 5km",
+                "max_download_speed": "Max available download speed (Mbps)",
             }
         )
     )
@@ -197,7 +196,7 @@ def main():
         "--noise", type=Path,  required=True, help="Road noise by postcode parquet file"
     )
     parser.add_argument(
-        "--ofsted", type=Path,  required=True, help="Ofsted school inspection parquet file"
+        "--school-proximity", type=Path, required=True, help="School proximity counts parquet file"
     )
     parser.add_argument(
         "--broadband", type=Path,  required=True, help="Broadband performance by output area parquet file"
@@ -216,7 +215,7 @@ def main():
         ethnicity_path=args.ethnicity,
         crime_path=args.crime,
         noise_path=args.noise,
-        ofsted_path=args.ofsted,
+        school_proximity_path=args.school_proximity,
         broadband_path=args.broadband,
     )