import zipfile
import httpx
import polars as pl
from pathlib import Path
from tqdm import tqdm

URL = "https://www.arcgis.com/sharing/rest/content/items/077631e063eb4e1ab43575d01381ec33/data"

BASE_DATA_PATH = Path("./data_sources")
BASE_DATA_PATH.mkdir(exist_ok=True)
DOWNLOAD_PATH = BASE_DATA_PATH / "arcgis_data.zip"
EXTRACT_PATH = BASE_DATA_PATH / "arcgis_extracted"
PARQUET_PATH = BASE_DATA_PATH / "arcgis_data.parquet"


def download_with_progress(url: str, output_path: Path) -> None:
    with httpx.stream(
        "GET",
        url,
        follow_redirects=True,
        timeout=httpx.Timeout(30.0, read=None),
    ) as response:
        response.raise_for_status()  # pyright: ignore[reportUnusedCallResult]
        total = int(response.headers.get("content-length", 0))

        with (
            open(output_path, "wb") as f,
            tqdm(
                total=total,
                unit="B",
                unit_scale=True,
                unit_divisor=1024,
                desc="Downloading",
            ) as pbar,
        ):
            for chunk in response.iter_bytes(chunk_size=8192):
                f.write(chunk)
                pbar.update(len(chunk))
        return 
   


def extract_zip(zip_path: Path, extract_path: Path) -> None:
    extract_path.mkdir(exist_ok=True)

    with zipfile.ZipFile(zip_path, "r") as zf:
        zf.extractall(extract_path)


def convert_to_parquet(data_path: Path, parquet_path: Path) -> None:
    df = pl.scan_csv(data_path / 'Data/NSPL_MAY_2025_UK.csv', try_parse_dates=True)
    print(f"Columns: {df.columns}")
    df.sink_parquet(parquet_path, compression="zstd")
    print(f"Saved to {parquet_path}")


def main() -> None:
    if PARQUET_PATH.exists():
        print(f"Parquet already exists at {PARQUET_PATH}, skipping")
        return

    if not DOWNLOAD_PATH.exists():
        download_with_progress(URL, DOWNLOAD_PATH)
    else:
        print(f"File already exists at {DOWNLOAD_PATH}, skipping download")

    extract_zip(DOWNLOAD_PATH, EXTRACT_PATH)
    convert_to_parquet(EXTRACT_PATH, PARQUET_PATH)

if __name__ == "__main__":
    main()