106 lines
3.7 KiB
Python
106 lines
3.7 KiB
Python
import argparse
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import polars as pl
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from pipeline.utils import download, extract_zip
|
|
|
|
# Ofcom Connected Nations 2025 - Fixed broadband performance (output area & local authority level)
|
|
# Source: https://www.ofcom.org.uk/phones-and-broadband/coverage-and-speeds/connected-nations-20252/data-downloads-2025
|
|
PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/research-and-data/multi-sector/infrastructure-research/connected-nations-2025/202507_fixed_broadband_coverage_r01.zip?v=407830"
|
|
|
|
# Pre-staged file path. Ofcom put the entire ofcom.org.uk domain behind
|
|
# Cloudflare's Managed Challenge in 2026, which requires a JS-executing
|
|
# browser to pass — no amount of User-Agent / TLS-impersonation spoofing
|
|
# (curl_cffi chrome120..131, safari17, firefox133, chrome_android) gets
|
|
# past it. When the automated download fails, the user must download the
|
|
# zip manually from the Source URL above and place it at this path.
|
|
MANUAL_ZIP_PATH = Path("manual-data/fixed_broadband_coverage.zip")
|
|
|
|
|
|
def _manual_download_instructions() -> str:
|
|
return (
|
|
f"\nOfcom has blocked automated downloads via Cloudflare's Managed\n"
|
|
f"Challenge. Download the zip manually and re-run:\n\n"
|
|
f" 1. Open in a browser:\n"
|
|
f" {PERFORMANCE_URL}\n"
|
|
f" 2. Save the downloaded zip to:\n"
|
|
f" {MANUAL_ZIP_PATH.resolve()}\n"
|
|
f" 3. Re-run `make -f Makefile.data property-data/broadband.parquet`\n"
|
|
)
|
|
|
|
|
|
def _obtain_zip(dest: Path) -> None:
|
|
"""Copy the pre-staged manual zip if present; otherwise attempt download."""
|
|
if MANUAL_ZIP_PATH.exists():
|
|
print(f"Using pre-staged zip: {MANUAL_ZIP_PATH}")
|
|
shutil.copyfile(MANUAL_ZIP_PATH, dest)
|
|
return
|
|
|
|
try:
|
|
download(PERFORMANCE_URL, dest)
|
|
except httpx.HTTPStatusError as e:
|
|
if e.response.status_code == 403:
|
|
print(_manual_download_instructions(), file=sys.stderr)
|
|
raise
|
|
|
|
|
|
def convert_to_parquet(extract_dir: Path, parquet_path: Path) -> None:
|
|
# Find CSV files in the extracted directory
|
|
csv_files = list(extract_dir.rglob("*.csv"))
|
|
if not csv_files:
|
|
raise FileNotFoundError(f"No CSV files found in {extract_dir}")
|
|
|
|
print(f"Found {len(csv_files)} CSV files: {[f.name for f in csv_files]}")
|
|
|
|
frames = []
|
|
for csv_file in sorted(csv_files):
|
|
print(f"Reading {csv_file.name}...")
|
|
df = pl.read_csv(csv_file, infer_schema_length=10000, encoding="utf8-lossy")
|
|
print(f" Shape: {df.shape}")
|
|
frames.append(df)
|
|
|
|
combined = pl.concat(frames, how="diagonal_relaxed")
|
|
print(f"Combined shape: {combined.shape}")
|
|
|
|
parquet_path.parent.mkdir(parents=True, exist_ok=True)
|
|
combined.write_parquet(parquet_path, compression="zstd")
|
|
print(f"Saved {parquet_path} ({combined.shape[0]} rows)")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download Ofcom broadband performance data"
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
required=True,
|
|
help="Output parquet file path",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
cache = Path(cache_dir)
|
|
zip_path = cache / "broadband_performance.zip"
|
|
extract_dir = cache / "extracted"
|
|
extracted_again_dir = cache / "extracted-again"
|
|
|
|
_obtain_zip(zip_path)
|
|
extract_zip(zip_path, extract_dir)
|
|
extract_zip(
|
|
extract_dir
|
|
/ "202507_fixed_coverage_r01"
|
|
/ "202507_fixed_pc_coverage_r01.zip",
|
|
extracted_again_dir,
|
|
)
|
|
|
|
convert_to_parquet(extracted_again_dir, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|