Extract common download utils
This commit is contained in:
parent
6ddb3d2121
commit
3b9ad11d71
9 changed files with 152 additions and 161 deletions
|
|
@ -1,47 +1,13 @@
|
|||
import argparse
|
||||
import tempfile
|
||||
import zipfile
|
||||
import httpx
|
||||
import polars as pl
|
||||
from pathlib import Path
|
||||
from tqdm import tqdm
|
||||
|
||||
from pipeline.utils import download, extract_zip
|
||||
|
||||
URL = "https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data"
|
||||
|
||||
|
||||
def download_with_progress(url: str, output_path: Path) -> None:
|
||||
with httpx.stream(
|
||||
"GET",
|
||||
url,
|
||||
follow_redirects=True,
|
||||
timeout=httpx.Timeout(30.0, read=None),
|
||||
) as response:
|
||||
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
|
||||
with (
|
||||
open(output_path, "wb") as f,
|
||||
tqdm(
|
||||
total=total,
|
||||
unit="B",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
desc="Downloading",
|
||||
) as pbar,
|
||||
):
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
return
|
||||
|
||||
|
||||
def extract_zip(zip_path: Path, extract_path: Path) -> None:
|
||||
extract_path.mkdir(exist_ok=True)
|
||||
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
zf.extractall(extract_path)
|
||||
|
||||
|
||||
def convert_to_parquet(data_path: Path, parquet_path: Path) -> None:
|
||||
df = pl.scan_csv(data_path / "Data/NSPL_MAY_2025_UK.csv", try_parse_dates=True)
|
||||
print(f"Columns: {df.collect_schema().names()}")
|
||||
|
|
@ -63,7 +29,7 @@ def main() -> None:
|
|||
download_path = Path(cache_dir) / "arcgis_data.zip"
|
||||
extract_path = Path(cache_dir) / "arcgis_extracted"
|
||||
|
||||
download_with_progress(URL, download_path)
|
||||
download(URL, download_path)
|
||||
extract_zip(download_path, extract_path)
|
||||
convert_to_parquet(extract_path, args.output)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,32 +1,13 @@
|
|||
import argparse
|
||||
import zipfile
|
||||
import httpx
|
||||
import tempfile
|
||||
import polars as pl
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from tqdm import tqdm
|
||||
|
||||
from pipeline.utils import download, extract_zip
|
||||
|
||||
# Ofcom Connected Nations 2025 - Fixed broadband performance (output area & local authority level)
|
||||
# Source: https://www.ofcom.org.uk/phones-and-broadband/coverage-and-speeds/connected-nations-20252/data-downloads-2025
|
||||
PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/research-and-data/multi-sector/infrastructure-research/connected-nations-2025/202507_fixed_broadband_performance_r01.zip"
|
||||
|
||||
|
||||
def download_with_progress(url: str, output_path: Path) -> None:
|
||||
with httpx.stream("GET", url, follow_redirects=True, timeout=120) as response:
|
||||
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
with (
|
||||
open(output_path, "wb") as f,
|
||||
tqdm(total=total, unit="B", unit_scale=True, desc="Downloading") as pbar,
|
||||
):
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
|
||||
|
||||
def extract_zip(zip_path: Path, extract_dir: Path) -> None:
|
||||
with zipfile.ZipFile(zip_path, "r") as z:
|
||||
z.extractall(extract_dir)
|
||||
PERFORMANCE_URL = "https://www.ofcom.org.uk/siteassets/resources/documents/research-and-data/multi-sector/infrastructure-research/connected-nations-2025/202507_fixed_broadband_coverage_r01.zip?v=407830"
|
||||
|
||||
|
||||
def convert_to_parquet(extract_dir: Path, parquet_path: Path) -> None:
|
||||
|
|
@ -35,22 +16,21 @@ def convert_to_parquet(extract_dir: Path, parquet_path: Path) -> None:
|
|||
if not csv_files:
|
||||
raise FileNotFoundError(f"No CSV files found in {extract_dir}")
|
||||
|
||||
print(f"Found CSV files: {[f.name for f in csv_files]}")
|
||||
print(f"Found {len(csv_files)} CSV files: {[f.name for f in csv_files]}")
|
||||
|
||||
# Read and concatenate all CSVs (typically split by geography level)
|
||||
frames = []
|
||||
for csv_file in sorted(csv_files):
|
||||
print(f"Reading {csv_file.name}...")
|
||||
df = pl.read_csv(csv_file, infer_schema_length=10000, encoding="utf8-lossy")
|
||||
print(f" Shape: {df.shape}, Columns: {df.columns}")
|
||||
frames.append((csv_file.stem, df))
|
||||
print(f" Shape: {df.shape}")
|
||||
frames.append(df)
|
||||
|
||||
# Save each CSV as a separate parquet file in the output directory
|
||||
parquet_path.mkdir(parents=True, exist_ok=True)
|
||||
for name, df in frames:
|
||||
out = parquet_path / f"{name}.parquet"
|
||||
df.write_parquet(out, compression="zstd")
|
||||
print(f"Saved {out} ({df.shape[0]} rows)")
|
||||
combined = pl.concat(frames, how="diagonal_relaxed")
|
||||
print(f"Combined shape: {combined.shape}")
|
||||
|
||||
parquet_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
combined.write_parquet(parquet_path, compression="zstd")
|
||||
print(f"Saved {parquet_path} ({combined.shape[0]} rows)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
|
@ -61,20 +41,22 @@ def main() -> None:
|
|||
"--output",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output directory for parquet files",
|
||||
help="Output parquet file path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
with tempfile.TemporaryDirectory(delete=False) as cache_dir:
|
||||
cache = Path(cache_dir)
|
||||
zip_path = cache / "broadband_performance.zip"
|
||||
extract_dir = cache / "extracted"
|
||||
extract_dir.mkdir()
|
||||
extracted_again_dir = cache / "extracted-again"
|
||||
|
||||
download_with_progress(PERFORMANCE_URL, zip_path)
|
||||
download(PERFORMANCE_URL, zip_path)
|
||||
extract_zip(zip_path, extract_dir)
|
||||
convert_to_parquet(extract_dir, args.output)
|
||||
print(list((extract_dir / "202507_fixed_coverage_r01").glob("*")))
|
||||
extract_zip(extract_dir / "202507_fixed_coverage_r01" / "202507_fixed_pc_coverage_r01.zip", extracted_again_dir)
|
||||
|
||||
convert_to_parquet(extracted_again_dir, args.output)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,29 +1,13 @@
|
|||
import argparse
|
||||
import httpx
|
||||
import tempfile
|
||||
import polars as pl
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
from pipeline.utils import download
|
||||
|
||||
URL = "https://assets.publishing.service.gov.uk/media/691ded34513046b952c500bd/File_5_IoD2025_Scores_for_the_Indices_of_Deprivation.xlsx"
|
||||
|
||||
|
||||
def download_file(url: str, output_path: Path) -> None:
|
||||
with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
|
||||
response.raise_for_status()
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
downloaded = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if total:
|
||||
print(
|
||||
f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB",
|
||||
end="",
|
||||
)
|
||||
print(f"\nSaved to {output_path}")
|
||||
|
||||
|
||||
def convert_to_parquet(xlsx_path: Path, parquet_path: Path) -> None:
|
||||
print("Reading Excel file (sheet 2)...")
|
||||
|
||||
|
|
@ -51,7 +35,7 @@ def main() -> None:
|
|||
|
||||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
xlsx_path = Path(cache_dir) / "IoD2025_Scores.xlsx"
|
||||
download_file(URL, xlsx_path)
|
||||
download(URL, xlsx_path, timeout=60)
|
||||
convert_to_parquet(xlsx_path, args.output)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,17 @@ import polars as pl
|
|||
NAPTAN_CSV_URL = "https://naptan.api.dft.gov.uk/v1/access-nodes?dataFormat=csv"
|
||||
|
||||
|
||||
STOP_TYPES = {
|
||||
'AIR': "Airport",
|
||||
'FTD': "Ferry",
|
||||
"RSE": "Rail station",
|
||||
"BCT": "Bus stop",
|
||||
"BCE": "Bus station",
|
||||
"TXR": "Taxi rank",
|
||||
"TMU": "Metro or Tram stop",
|
||||
}
|
||||
|
||||
|
||||
def download_naptan(output: Path) -> None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
|
@ -26,10 +37,11 @@ def download_naptan(output: Path) -> None:
|
|||
pl.col("Longitude").cast(pl.Float64, strict=False),
|
||||
)
|
||||
.drop_nulls(subset=["Latitude", "Longitude"])
|
||||
.filter(pl.col("StopType").is_in(list(STOP_TYPES.keys())))
|
||||
.select(
|
||||
pl.col("ATCOCode").alias("id"),
|
||||
pl.col("CommonName").alias("name"),
|
||||
pl.col("StopType").alias("category"),
|
||||
pl.col("StopType").replace(STOP_TYPES).alias("category"),
|
||||
pl.col("Latitude").alias("lat"),
|
||||
pl.col("Longitude").alias("lng"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,31 +1,15 @@
|
|||
import argparse
|
||||
import httpx
|
||||
import tempfile
|
||||
import polars as pl
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
from pipeline.utils import download
|
||||
|
||||
# Management information - state-funded schools - latest inspections (as at 30 Apr 2025)
|
||||
# Source: https://www.gov.uk/government/statistical-data-sets/monthly-management-information-ofsteds-school-inspections-outcomes
|
||||
URL = "https://assets.publishing.service.gov.uk/media/681cd390275cb67b18d870fc/Management_information_-_state-funded_schools_-_latest_inspections_as_at_30_Apr_2025.csv"
|
||||
|
||||
|
||||
def download_file(url: str, output_path: Path) -> None:
|
||||
with httpx.stream("GET", url, follow_redirects=True, timeout=60) as response:
|
||||
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
downloaded = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if total:
|
||||
print(
|
||||
f"\rDownloaded {downloaded / 1024 / 1024:.1f} MB / {total / 1024 / 1024:.1f} MB",
|
||||
end="",
|
||||
)
|
||||
print(f"\nSaved to {output_path}")
|
||||
|
||||
|
||||
def convert_to_parquet(csv_path: Path, parquet_path: Path) -> None:
|
||||
print("Reading CSV...")
|
||||
|
||||
|
|
@ -54,7 +38,7 @@ def main() -> None:
|
|||
|
||||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
csv_path = Path(cache_dir) / "ofsted_latest_inspections.csv"
|
||||
download_file(URL, csv_path)
|
||||
download(URL, csv_path, timeout=60)
|
||||
convert_to_parquet(csv_path, args.output)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +1,13 @@
|
|||
import argparse
|
||||
import tempfile
|
||||
import httpx
|
||||
import polars as pl
|
||||
from pathlib import Path
|
||||
from tqdm import tqdm
|
||||
|
||||
from pipeline.utils import download
|
||||
|
||||
URL = "http://prod.publicdata.landregistry.gov.uk.s3-website-eu-west-1.amazonaws.com/pp-complete.csv"
|
||||
|
||||
|
||||
def download_with_progress(url: str, output_path: Path) -> None:
|
||||
with httpx.stream(
|
||||
"GET",
|
||||
url,
|
||||
follow_redirects=True,
|
||||
timeout=httpx.Timeout(30.0, read=None),
|
||||
) as response:
|
||||
response.raise_for_status() # pyright: ignore[reportUnusedCallResult]
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
|
||||
with (
|
||||
open(output_path, "wb") as f,
|
||||
tqdm(
|
||||
total=total,
|
||||
unit="B",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
desc="Downloading",
|
||||
) as pbar,
|
||||
):
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
return
|
||||
|
||||
|
||||
def convert_to_parquet(csv_path: Path, parquet_path: Path) -> None:
|
||||
"""Convert CSV to Parquet using Polars."""
|
||||
print("Converting to Parquet...")
|
||||
|
|
@ -84,7 +58,7 @@ def main() -> None:
|
|||
with tempfile.TemporaryDirectory() as cache_dir:
|
||||
csv_path = Path(cache_dir) / "price-paid-complete.csv"
|
||||
|
||||
download_with_progress(URL, csv_path)
|
||||
download(URL, csv_path)
|
||||
convert_to_parquet(csv_path, args.output)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue