Extract common download utils

pipeline/transform/merge.py

@@ -6,11 +6,14 @@ from pathlib import Path
 def _build_wide(
     epc_pp_path: Path,
     arcgis_path: Path,
-    iod_path: Path | None,
-    poi_proximity_path: Path | None,
-    journey_times_path: Path | None,
-    ethnicity_path: Path | None,
-    crime_path: Path | None,
+    iod_path: Path,
+    poi_proximity_path: Path,
+    journey_times_path: Path,
+    ethnicity_path: Path,
+    crime_path: Path ,
+    noise_path: Path ,
+    ofsted_path: Path,
+    broadband_path: Path,
 ) -> pl.DataFrame:
     """Build the wide dataframe by joining epc_pp with all auxiliary data."""
     print("Scanning epc_pp...")
@@ -23,19 +26,18 @@ def _build_wide(
         "lat",
         pl.col("long").alias("lon"),
         "lsoa21",
+        "oa21",
     )
     wide = wide.join(arcgis, on="postcode", how="inner")
 
-    # Journey times (optional)
-    if journey_times_path and journey_times_path.exists():
-        print("Joining journey times...")
-        journey_times = pl.scan_parquet(journey_times_path).select(
-            "postcode",
-            "public_transport_easy_minutes",
-            "public_transport_quick_minutes",
-            "cycling_minutes",
-        )
-        wide = wide.join(journey_times, on="postcode", how="left")
+    print("Joining journey times...")
+    journey_times = pl.scan_parquet(journey_times_path).select(
+        "postcode",
+        "public_transport_easy_minutes",
+        "public_transport_quick_minutes",
+        "cycling_minutes",
+    )
+    wide = wide.join(journey_times, on="postcode", how="left")
 
     print("Joining IoD scores...")
     iod = pl.scan_parquet(iod_path)
@@ -60,6 +62,32 @@ def _build_wide(
     poi_counts = pl.scan_parquet(poi_proximity_path)
     wide = wide.join(poi_counts, on="postcode", how="left")
 
+    noise = pl.scan_parquet(noise_path).select("postcode", "road_noise_lden_db")
+    wide = wide.join(noise, on="postcode", how="left")
+
+    print("Joining Ofsted school ratings...")
+    ofsted = (
+        pl.scan_parquet(ofsted_path)
+        .filter(pl.col("Overall effectiveness").is_in(["1", "2", "3", "4"]))
+        .select(
+            pl.col("Postcode").alias("ofsted_postcode"),
+            pl.col("Overall effectiveness").cast(pl.UInt8).alias("ofsted_rating"),
+        )
+        .group_by("ofsted_postcode")
+        .agg(pl.col("ofsted_rating").mean().round(1).alias("ofsted_avg_rating"))
+    )
+    wide = wide.join(ofsted, left_on="postcode", right_on="ofsted_postcode", how="left")
+
+    print("Joining broadband performance...")
+    broadband = pl.scan_parquet(broadband_path).select(
+        "output_area",
+        pl.col("Average max download speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_download_900plus"),
+        pl.col("Average max download speed (Mbit/s) for lines 100<300Mbit/s").alias("broadband_max_download_100_300"),
+        pl.col("Average max download speed (Mbit/s) for lines 30<100Mbit/s").alias("broadband_max_download_30_100"),
+        pl.col("Average max upload speed (Mbit/s) for lines >=900Mbit/s").alias("broadband_max_upload_900plus"),
+    )
+    wide = wide.join(broadband, left_on="oa21", right_on="output_area", how="left")
+
     # Convert construction_age_band to numeric year
     wide = wide.with_columns(
         pl.col("construction_age_band")
@@ -103,6 +131,7 @@ def _build_wide(
             "Income Deprivation Affecting Children Index (IDACI) Score (rate)",
             "Barriers to Housing and Services Score",
             "lsoa21",
+            "oa21",
             "pp_property_type",
             "built_form",
         )
@@ -124,6 +153,12 @@ def _build_wide(
                 "public_transport_2km": "Public transport within 2km",
                 "latest_price": "Last known price",
                 "number_habitable_rooms": "Rooms (including bedrooms & bathrooms)",
+                "road_noise_lden_db": "Road noise Lden (dB)",
+                "ofsted_avg_rating": "Ofsted avg rating (1=Outstanding, 4=Inadequate)"
+                        "broadband_max_download_900plus": "Broadband download speed 900+ Mbps",
+                        "broadband_max_download_100_300": "Broadband download speed 100-300 Mbps",
+                        "broadband_max_download_30_100": "Broadband download speed 30-100 Mbps",
+                        "broadband_max_upload_900plus": "Broadband upload speed 900+ Mbps",
             }
         )
     )
@@ -131,7 +166,6 @@ def _build_wide(
     print("Collecting with streaming engine...")
     return wide.collect(engine="streaming")
 
-
 def main():
     parser = argparse.ArgumentParser(
         description="Build wide property dataframe with all joins"
@@ -159,6 +193,15 @@ def main():
     parser.add_argument(
         "--crime", type=Path, required=True, help="Crime by LSOA parquet file (optional)"
     )
+    parser.add_argument(
+        "--noise", type=Path,  required=True, help="Road noise by postcode parquet file"
+    )
+    parser.add_argument(
+        "--ofsted", type=Path,  required=True, help="Ofsted school inspection parquet file"
+    )
+    parser.add_argument(
+        "--broadband", type=Path,  required=True, help="Broadband performance by output area parquet file"
+    )
     parser.add_argument(
         "--output", type=Path, required=True, help="Output parquet file path"
     )
@@ -172,6 +215,9 @@ def main():
         journey_times_path=args.journey_times,
         ethnicity_path=args.ethnicity,
         crime_path=args.crime,
+        noise_path=args.noise,
+        ofsted_path=args.ofsted,
+        broadband_path=args.broadband,
     )
 
     print(f"Columns: {wide.columns}")