66 lines
1.6 KiB
Python
66 lines
1.6 KiB
Python
import argparse
|
|
import tempfile
|
|
import polars as pl
|
|
from pathlib import Path
|
|
|
|
from pipeline.utils import download
|
|
|
|
URL = "http://prod.publicdata.landregistry.gov.uk.s3-website-eu-west-1.amazonaws.com/pp-complete.csv"
|
|
|
|
|
|
def convert_to_parquet(csv_path: Path, parquet_path: Path) -> None:
|
|
"""Convert CSV to Parquet using Polars."""
|
|
print("Converting to Parquet...")
|
|
|
|
# https://www.gov.uk/guidance/about-the-price-paid-data
|
|
# Land Registry CSV columns
|
|
columns = [
|
|
"transaction_id",
|
|
"price",
|
|
"date_of_transfer",
|
|
"postcode",
|
|
"property_type",
|
|
"old_new",
|
|
"duration",
|
|
"paon",
|
|
"saon",
|
|
"street",
|
|
"locality",
|
|
"town_city",
|
|
"district",
|
|
"county",
|
|
"ppd_category",
|
|
"record_status",
|
|
]
|
|
|
|
df = pl.read_csv(
|
|
csv_path,
|
|
has_header=False,
|
|
new_columns=columns,
|
|
try_parse_dates=True,
|
|
)
|
|
|
|
parquet_path.parent.mkdir(parents=True, exist_ok=True)
|
|
print(f"Columns: {df.collect_schema().names()}")
|
|
df.write_parquet(parquet_path, compression="zstd")
|
|
print(f"Saved to {parquet_path}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Download and convert Land Registry price-paid data"
|
|
)
|
|
parser.add_argument(
|
|
"--output", type=Path, required=True, help="Output parquet file path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as cache_dir:
|
|
csv_path = Path(cache_dir) / "price-paid-complete.csv"
|
|
|
|
download(URL, csv_path)
|
|
convert_to_parquet(csv_path, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|