80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
import argparse
|
|
import zipfile
|
|
import httpx
|
|
import polars as pl
|
|
from pathlib import Path
|
|
import tempfile
|
|
from tqdm import tqdm
|
|
|
|
# Ofcom Connected Nations 2025 - Fixed broadband performance (output area & local authority level)
|
|
# Source: https://www.ofcom.org.uk/phones-and-broadband/coverage-and-speeds/connected-nations-20252/data-downloads-2025
|
|
PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/research-and-data/multi-sector/infrastructure-research/connected-nations-2025/202507_fixed_broadband_performance_r01.zip"
|
|
|
|
|
|
def download_with_progress(url: str, output_path: Path) -> None:
|
|
with httpx.stream("GET", url, follow_redirects=True, timeout=120) as response:
|
|
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
|
total = int(response.headers.get("content-length", 0))
|
|
with (
|
|
open(output_path, "wb") as f,
|
|
tqdm(total=total, unit="B", unit_scale=True, desc="Downloading") as pbar,
|
|
):
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
f.write(chunk)
|
|
pbar.update(len(chunk))
|
|
|
|
|
|
def extract_zip(zip_path: Path, extract_dir: Path) -> None:
|
|
with zipfile.ZipFile(zip_path, "r") as z:
|
|
z.extractall(extract_dir)
|
|
|
|
|
|
def convert_to_parquet(extract_dir: Path, parquet_path: Path) -> None:
|
|
# Find CSV files in the extracted directory
|
|
csv_files = list(extract_dir.rglob("*.csv"))
|
|
if not csv_files:
|
|
raise FileNotFoundError(f"No CSV files found in {extract_dir}")
|
|
|
|
print(f"Found CSV files: {[f.name for f in csv_files]}")
|
|
|
|
# Read and concatenate all CSVs (typically split by geography level)
|
|
frames = []
|
|
for csv_file in sorted(csv_files):
|
|
print(f"Reading {csv_file.name}...")
|
|
df = pl.read_csv(csv_file, infer_schema_length=10000, encoding="utf8-lossy")
|
|
print(f" Shape: {df.shape}, Columns: {df.columns}")
|
|
frames.append((csv_file.stem, df))
|
|
|
|
# Save each CSV as a separate parquet file in the output directory
|
|
parquet_path.mkdir(parents=True, exist_ok=True)
|
|
for name, df in frames:
|
|
out = parquet_path / f"{name}.parquet"
|
|
df.write_parquet(out, compression="zstd")
|
|
print(f"Saved {out} ({df.shape[0]} rows)")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download Ofcom broadband performance data"
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
required=True,
|
|
help="Output directory for parquet files",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
cache = Path(cache_dir)
|
|
zip_path = cache / "broadband_performance.zip"
|
|
extract_dir = cache / "extracted"
|
|
extract_dir.mkdir()
|
|
|
|
download_with_progress(PERFORMANCE_URL, zip_path)
|
|
extract_zip(zip_path, extract_dir)
|
|
convert_to_parquet(extract_dir, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|